Skip to content
Snippets Groups Projects
Commit 4eb2bbb0 authored by Rainer Kartmann's avatar Rainer Kartmann
Browse files

Merge branch 'armem/dev' into armem/robot-state-memory

parents 539e3860 7fe56c22
No related branches found
No related tags found
No related merge requests found
Showing
with 195 additions and 64 deletions
......@@ -24,6 +24,10 @@
#include <mutex>
#include <thread>
#include <string>
#include <vector>
#include <mutex>
#include <map>
#include <ArmarXCore/core/Component.h>
......
......@@ -13,15 +13,16 @@ module armarx
{
enum QueryTarget
{
WMAndLTM,
WM,
LTM // only for debug. remove later
LTM,
Disk
};
sequence<QueryTarget> QueryTargets;
/// Which entity snapshots to get from an entity?
class EntityQuery
{
QueryTarget target = QueryTarget::WMAndLTM;
QueryTargets targets;
};
sequence<EntityQuery> EntityQuerySeq;
......@@ -115,7 +116,7 @@ module armarx
class ProviderSegmentQuery
{
EntityQuerySeq entityQueries;
QueryTarget target = QueryTarget::WMAndLTM;
QueryTargets targets;
};
sequence<ProviderSegmentQuery> ProviderSegmentQuerySeq;
module provider
......@@ -138,7 +139,7 @@ module armarx
class CoreSegmentQuery
{
ProviderSegmentQuerySeq providerSegmentQueries;
QueryTarget target = QueryTarget::WMAndLTM;
QueryTargets targets;
};
sequence<CoreSegmentQuery> CoreSegmentQuerySeq;
......@@ -162,7 +163,7 @@ module armarx
class MemoryQuery
{
CoreSegmentQuerySeq coreSegmentQueries;
QueryTarget target = QueryTarget::WMAndLTM;
QueryTargets targets;
};
sequence<MemoryQuery> MemoryQuerySeq;
dictionary<string, MemoryQuerySeq> MemoryQueriesDict;
......@@ -188,7 +189,6 @@ module armarx
{
/// Dict of memory name to
MemoryQueriesDict memoryQueries;
QueryTarget target = QueryTarget::WMAndLTM;
};
struct Input
......
......@@ -122,6 +122,12 @@ set(LIB_FILES
server/query_proc/longtermmemory/CoreSegmentQueryProcessor.cpp
server/query_proc/longtermmemory/MemoryQueryProcessor.cpp
server/query_proc/diskmemory/BaseQueryProcessor.cpp
server/query_proc/diskmemory/EntityQueryProcessor.cpp
server/query_proc/diskmemory/ProviderSegmentQueryProcessor.cpp
server/query_proc/diskmemory/CoreSegmentQueryProcessor.cpp
server/query_proc/diskmemory/MemoryQueryProcessor.cpp
mns/MemoryNameSystem.cpp
mns/ComponentPlugin.cpp
......@@ -238,6 +244,12 @@ set(LIB_HEADERS
server/query_proc/longtermmemory/CoreSegmentQueryProcessor.h
server/query_proc/longtermmemory/MemoryQueryProcessor.h
server/query_proc/diskmemory/BaseQueryProcessor.h
server/query_proc/diskmemory/EntityQueryProcessor.h
server/query_proc/diskmemory/ProviderSegmentQueryProcessor.h
server/query_proc/diskmemory/CoreSegmentQueryProcessor.h
server/query_proc/diskmemory/MemoryQueryProcessor.h
mns.h
mns/MemoryNameSystem.h
mns/ComponentPlugin.h
......
......@@ -28,6 +28,37 @@ namespace armarx::armem::client
armem::query::data::MemoryQuerySeq memoryQueries;
DataMode dataMode;
void addQueryTargetToAll(const armem::query::data::QueryTarget t)
{
for (const auto& memoryQuery : memoryQueries)
{
if (std::find(memoryQuery->targets.begin(), memoryQuery->targets.end(), t) == memoryQuery->targets.end())
{
memoryQuery->targets.push_back(t);
}
for (const auto& coreSegmentQuery : memoryQuery->coreSegmentQueries)
{
if (std::find(coreSegmentQuery->targets.begin(), coreSegmentQuery->targets.end(), t) == coreSegmentQuery->targets.end())
{
coreSegmentQuery->targets.push_back(t);
}
for (const auto& providerSegmentQuery : coreSegmentQuery->providerSegmentQueries)
{
if (std::find(providerSegmentQuery->targets.begin(), providerSegmentQuery->targets.end(), t) == providerSegmentQuery->targets.end())
{
providerSegmentQuery->targets.push_back(t);
}
for (const auto& entityQuery : providerSegmentQuery->entityQueries)
{
if (std::find(entityQuery->targets.begin(), entityQuery->targets.end(), t) == entityQuery->targets.end())
{
entityQuery->targets.push_back(t);
}
}
}
}
}
}
static QueryInput fromIce(const armem::query::data::Input& ice);
armem::query::data::Input toIce() const;
......
......@@ -8,12 +8,6 @@ namespace armarx::armem::client::query
{
}
Builder& Builder::queryTarget(const armem::query::data::QueryTarget target)
{
this->_target = target;
return *this;
}
QueryInput Builder::buildQueryInput() const
{
QueryInput input;
......@@ -31,9 +25,9 @@ namespace armarx::armem::client::query
armem::query::data::MemoryQuerySeq memoryQueries;
for (const CoreSegmentSelector& child : _children)
{
for (const armem::query::data::MemoryQueryPtr& query : child.buildQueries())
for (armem::query::data::MemoryQueryPtr& query : child.buildQueries())
{
query->target = _target;
query->targets = _targets;
memoryQueries.push_back(query);
}
}
......
......@@ -28,7 +28,6 @@ namespace armarx::armem::client::query
public:
Builder(DataMode dataMode = DataMode::WithData);
Builder& queryTarget(const armem::query::data::QueryTarget target);
/// Start specifying core segments.
CoreSegmentSelector& coreSegments();
......@@ -66,7 +65,6 @@ namespace armarx::armem::client::query
public:
armem::query::data::QueryTarget _target = armem::query::data::QueryTarget::WMAndLTM;
DataMode dataMode;
};
......
......@@ -5,6 +5,7 @@
#include <Ice/Handle.h>
#include <RobotAPI/libraries/armem/core/MemoryID.h>
#include <RobotAPI/interface/armem/query.h>
namespace armarx::armem::client::query::detail
......@@ -17,7 +18,11 @@ namespace armarx::armem::client::query::detail
using DerivedT = _DerivedT;
using QueryT = _QueryT;
virtual ~ChildSelectorOps() = default;
ChildSelectorOps() = delete;
ChildSelectorOps(const armem::query::data::QueryTargets& p) :
_parentTargets(p)
{}
virtual DerivedT& all() = 0;
......@@ -36,6 +41,9 @@ namespace armarx::armem::client::query::detail
}
}
protected:
armem::query::data::QueryTargets _parentTargets;
protected:
......@@ -81,11 +89,21 @@ namespace armarx::armem::client::query::detail
using ChildT = _ChildT;
public:
virtual ~ParentSelectorOps() = default;
/// QueryTargets for this query
DerivedT& queryTargets(const armem::query::data::QueryTargets& targets)
{
this->_targets = targets;
return dynamic_cast<DerivedT&>(*this);
}
protected:
ChildT& _addChild()
{
return _children.emplace_back();
return _children.emplace_back(_targets);
}
ChildT& _addChild(const ChildT& child)
{
......@@ -102,7 +120,7 @@ namespace armarx::armem::client::query::detail
std::vector<ChildT> _children;
armem::query::data::QueryTargets _targets = {armem::query::data::QueryTarget::WM}; // if not specified we only query the WM
};
......@@ -114,17 +132,27 @@ namespace armarx::armem::client::query::detail
{
public:
virtual ~InnerSelectorOps()
InnerSelectorOps(const armem::query::data::QueryTargets& p) :
ChildSelectorOps<DerivedT, QueryT>(p)
{}
virtual std::vector<QueryT> buildQueries() const
{
// check own query targets and parent query targets
for (const auto& target : this->_targets)
{
if (std::find(this->_parentTargets.begin(), this->_parentTargets.end(), target) == this->_parentTargets.end())
{
// TODO: Error, this makes no sense! or just ignore?
}
}
std::vector<typename ChildT::QueryT> childQueries;
for (const auto& child : this->_children)
{
for (const auto& query : child.buildQueries())
for (auto& query : child.buildQueries())
{
query->targets = this->_targets;
childQueries.push_back(query);
}
}
......
......@@ -77,7 +77,6 @@ namespace armarx::armem::client::query
return *this;
}
SnapshotSelector& EntitySelector::snapshots()
{
return _addChild();
......@@ -115,7 +114,6 @@ namespace armarx::armem::client::query
}
EntitySelector& ProviderSegmentSelector::entities()
{
return _addChild();
......@@ -153,7 +151,6 @@ namespace armarx::armem::client::query
}
ProviderSegmentSelector& CoreSegmentSelector::providerSegments()
{
return _addChild();
......
......@@ -15,6 +15,10 @@ namespace armarx::armem::client::query
{
public:
SnapshotSelector(const armem::query::data::QueryTargets& p) :
detail::ChildSelectorOps<SnapshotSelector, armem::query::data::EntityQueryPtr>(p)
{}
armem::query::data::EntityQuerySeq buildQueries() const;
......@@ -44,6 +48,10 @@ namespace armarx::armem::client::query
{
public:
EntitySelector(const armem::query::data::QueryTargets& p) :
detail::InnerSelectorOps<EntitySelector, armem::query::data::ProviderSegmentQueryPtr, SnapshotSelector>(p)
{}
/// Start specifying entity snapshots.
SnapshotSelector& snapshots();
SnapshotSelector& snapshots(const SnapshotSelector& selector);
......@@ -79,6 +87,10 @@ namespace armarx::armem::client::query
{
public:
ProviderSegmentSelector(const armem::query::data::QueryTargets& p) :
detail::InnerSelectorOps<ProviderSegmentSelector, armem::query::data::CoreSegmentQueryPtr, EntitySelector>(p)
{}
/// Start specifying entities.
EntitySelector& entities();
EntitySelector& entities(const EntitySelector& selector);
......@@ -114,6 +126,10 @@ namespace armarx::armem::client::query
{
public:
CoreSegmentSelector(const armem::query::data::QueryTargets& p) :
detail::InnerSelectorOps<CoreSegmentSelector, armem::query::data::MemoryQueryPtr, ProviderSegmentSelector>(p)
{}
/// Start specifying provider segments.
ProviderSegmentSelector& providerSegments();
ProviderSegmentSelector& providerSegments(const ProviderSegmentSelector& selector);
......
......@@ -24,7 +24,7 @@ namespace armarx::armem::d_ltm
wm::CoreSegment CoreSegment::convert() const
{
wm::CoreSegment m;
wm::CoreSegment m(id());
for (const auto& [_, s] : _container)
{
m.addProviderSegment(s.convert(_aronType));
......@@ -86,7 +86,16 @@ namespace armarx::armem::d_ltm
}
else
{
std::filesystem::create_directory(_fullPath() / k);
try
{
std::filesystem::create_directory(_fullPath() / k);
}
catch (...)
{
ARMARX_WARNING << GetHandledExceptionString();
return;
}
auto wms = _container.emplace(std::make_pair(k, id().withProviderSegmentName(k)));
wms.first->second.path = path;
wms.first->second.append(s);
......
......@@ -18,7 +18,7 @@ namespace armarx::armem::d_ltm
wm::Entity Entity::convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const
{
wm::Entity m;
wm::Entity m(id());
for (const auto& [_, s] : _container)
{
m.addSnapshot(s.convert(expectedStructure));
......@@ -67,7 +67,8 @@ namespace armarx::armem::d_ltm
{
if (const auto& it = _container.find(k); it != _container.end())
{
it->second.setTo(s);
// timestamp already exists
// We assume that a snapshot does not change, so ignore
}
else
{
......
......@@ -41,6 +41,7 @@ namespace armarx::armem::d_ltm
void EntityInstance::_copySelf(EntityInstance& other) const
{
EntityInstanceBase<EntityInstance>::_copySelf(other);
other.path = path;
}
std::filesystem::path EntityInstance::_fullPath() const
......@@ -49,6 +50,7 @@ namespace armarx::armem::d_ltm
{
return _fullPath(*path);
}
ARMARX_WARNING << "The path of the disk memory instance with id '" << id().str() << "' is not set. This may lead to errors.";
return std::filesystem::path();
}
......@@ -59,7 +61,7 @@ namespace armarx::armem::d_ltm
wm::EntityInstance EntityInstance::convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const
{
std::filesystem::path p = _fullPath();
std::filesystem::path p = _fullPath(); // here we assume that "reload" has been called first
std::filesystem::path d = p / (std::string(DATA_FILENAME) + ".json");
if (std::filesystem::is_regular_file(d))
......@@ -67,7 +69,7 @@ namespace armarx::armem::d_ltm
std::ifstream ifs(d);
std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
nlohmann::json j(file_content);
nlohmann::json j = nlohmann::json::parse(file_content);
auto aron = std::make_shared<aron::datanavigator::DictNavigator>();
to_aron(aron, j, expectedStructure);
wm::EntityInstance e(id());
......@@ -76,7 +78,7 @@ namespace armarx::armem::d_ltm
}
else
{
throw error::ArMemError("An diskMemory EntityInstance is not leading to a regular file.");
throw error::ArMemError("An diskMemory EntityInstance is not leading to a regular file. The path was: " + d.string());
}
}
......@@ -84,7 +86,7 @@ namespace armarx::armem::d_ltm
{
if (!p_ptr)
{
ARMARX_WARNING << "The entered is NULL.";
ARMARX_WARNING << "The entered path is NULL.";
}
std::filesystem::path p = _fullPath(*p_ptr);
if (!std::filesystem::is_directory(p))
......
......@@ -26,7 +26,7 @@ namespace armarx::armem::d_ltm
wm::EntitySnapshot EntitySnapshot::convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const
{
wm::EntitySnapshot m;
wm::EntitySnapshot m(id());
for (const auto& s : _container)
{
m.addInstance(s.convert(expectedStructure));
......@@ -56,7 +56,7 @@ namespace armarx::armem::d_ltm
std::filesystem::path d = p / std::to_string(i);
if (std::filesystem::is_directory(d))
{
auto wms = _container.emplace_back(id().withInstanceIndex(i));
auto& wms = _container.emplace_back(id().withInstanceIndex(i));
wms.reload(p_ptr);
}
else
......@@ -79,7 +79,7 @@ namespace armarx::armem::d_ltm
return;
}
// We remove the contente here and reset it with new values
// We remove the content here and reset it with new values
_container.clear();
int i = 0;
......@@ -95,7 +95,7 @@ namespace armarx::armem::d_ltm
continue;;
}
auto wms = _container.emplace_back(id().withInstanceIndex(i++));
auto& wms = _container.emplace_back(id().withInstanceIndex(i++));
wms.path = path;
wms.setTo(s);
}
......
......@@ -29,7 +29,7 @@ namespace armarx::armem::d_ltm
void reload(const std::shared_ptr<std::filesystem::path>&);
wm::EntitySnapshot convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const;
// MongoDB connection
// FS connection
void setTo(const wm::EntitySnapshot&);
private:
......
......@@ -24,8 +24,8 @@ namespace armarx::armem::d_ltm
wm::Memory Memory::convert() const
{
wm::Memory m;
for (const auto& [_, s] : _container)
wm::Memory m(id());
for (const auto& [k, s] : _container)
{
m.addCoreSegment(s.convert());
}
......@@ -44,7 +44,7 @@ namespace armarx::armem::d_ltm
if (!std::filesystem::exists(p))
{
ARMARX_INFO << "The entered path does not exist. Assuming an empty container.";
ARMARX_INFO << "The entered path does not exist. Returning empty memory.";
}
else
{
......@@ -73,7 +73,15 @@ namespace armarx::armem::d_ltm
}
else
{
std::filesystem::create_directory(_fullPath() / k);
try
{
std::filesystem::create_directory(_fullPath() / k);
}
catch (...)
{
ARMARX_WARNING << GetHandledExceptionString();
return;
}
auto wms = _container.emplace(std::make_pair(k, id().withCoreSegmentName(k)));
wms.first->second.path = path;
......
......@@ -23,7 +23,7 @@ namespace armarx::armem::d_ltm
wm::ProviderSegment ProviderSegment::convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const
{
wm::ProviderSegment m;
wm::ProviderSegment m(id());
for (const auto& [_, s] : _container)
{
if (hasAronType())
......@@ -107,8 +107,6 @@ namespace armarx::armem::d_ltm
try
{
std::filesystem::create_directory(_fullPath() / k);
continue;
}
catch (...)
{
......
......@@ -64,7 +64,7 @@ namespace armarx::armem::ltm
{
if (const auto& it = _container.find(k); it != _container.end())
{
// segment already exists
// timestamp already exists
// We assume that a snapshot does not change, so ignore
}
else
......
......@@ -29,7 +29,6 @@ namespace armarx::armem::server
data::AddSegmentResult
MemoryToIceAdapter::addSegment(const data::AddSegmentInput& input, bool addCoreSegments)
{
ARMARX_DEBUG << "Adding segment '" << input.coreSegmentName << "/" << input.providerSegmentName << "'.";
ARMARX_CHECK_NOT_NULL(workingMemory);
data::AddSegmentResult output;
......@@ -206,10 +205,10 @@ namespace armarx::armem::server
: armem::DataMode::NoData);
armem::query::data::Result result;
wm::Memory wm_result = wm_processor.process(input, *workingMemory, /* ignore if: */ { query::data::QueryTarget::LTM });
wm::Memory wm_result = wm_processor.process(input, *workingMemory, /* execute if: */ { query::data::QueryTarget::WM });
armem::ltm::query_proc::MemoryQueryProcessor ltm_processor;
ltm::Memory ltm_result = ltm_processor.process(input, *longtermMemory, /* ignore if: */ { query::data::QueryTarget::WM });
ltm::Memory ltm_result = ltm_processor.process(input, *longtermMemory, /* execute if: */ { query::data::QueryTarget::LTM });
if (ltm_result.hasData())
......@@ -238,6 +237,11 @@ namespace armarx::armem::server
}
result.success = true;
if (result.memory->coreSegments.size() == 0)
{
ARMARX_DEBUG << "No data in memory found after query.";
}
return result;
}
......@@ -262,7 +266,7 @@ namespace armarx::armem::server
{
// force query to only query WM
// TODO: Think about other query class (fabian.peller)
query->target = query::data::QueryTarget::WM;
query->targets = {query::data::QueryTarget::WM};
}
armem::query::data::Result queryResult = this->query(input.query);
......
#pragma once
#include <ArmarXCore/core/logging/Logging.h>
#include <RobotAPI/interface/armem/query.h>
......@@ -18,39 +19,67 @@ namespace armarx::armem::base::query_proc
public:
DataT process(const QueryT& query, const DataT& data, const std::vector<query::data::QueryTarget>& ignoreTargets = {}) const
DataT process(const QueryT& query, const DataT& data, const query::data::QueryTargets& executeIf = {}) const
{
DataT result = data.copyEmpty();
if (std::find(ignoreTargets.begin(), ignoreTargets.end(), query.target) != ignoreTargets.end())
for (const auto queryTarget : query.targets)
{
return result;
if (std::find(executeIf.begin(), executeIf.end(), queryTarget) != executeIf.end())
{
this->process(result, query, data);
break;
}
}
this->process(result, query, data);
return result;
}
DataT process(const QueryPtrT& query, const DataT& data, const std::vector<query::data::QueryTarget>& ignoreTargets = {}) const
DataT process(const QueryPtrT& query, const DataT& data, const query::data::QueryTargets& executeIf = {}) const
{
return this->process(*query, *data, ignoreTargets);
return this->process(*query, *data, executeIf);
}
DataT process(const QuerySeqT& queries, const DataT& data, const std::vector<query::data::QueryTarget>& ignoreTargets = {}) const
DataT process(const QuerySeqT& queries, const DataT& data, const query::data::QueryTargets& executeIf = {}) const
{
DataT result = data.copyEmpty();
this->process(result, queries, data, ignoreTargets);
this->process(result, queries, data, executeIf);
return result;
}
void process(DataT& result, const QuerySeqT& queries, const DataT& data, const std::vector<query::data::QueryTarget>& ignoreTargets = {}) const
void process(DataT& result, const QuerySeqT& queries, const DataT& data, const query::data::QueryTargets& executeIf = {}) const
{
if (queries.empty())
{
ARMARX_DEBUG << "There are no queries to process.";
return;
}
if (executeIf.empty())
{
ARMARX_DEBUG << "Could not execute query. ExecuteIf s empty.";
return;
}
for (const auto& query : queries)
{
if (std::find(ignoreTargets.begin(), ignoreTargets.end(), query->target) != ignoreTargets.end())
if (query->targets.empty())
{
ARMARX_DEBUG << "The targets of a query are empty";
continue;
}
this->process(result, *query, data);
for (const auto queryTarget : query->targets)
{
if (std::find(executeIf.begin(), executeIf.end(), queryTarget) != executeIf.end())
{
this->process(result, *query, data);
break;
}
else
{
ARMARX_DEBUG << "The query target " << queryTarget << " was not found in executeIf: " << executeIf;
}
}
}
}
......
......@@ -34,9 +34,9 @@ namespace armarx::armem::base::query_proc
using Base::process;
_MemoryT process(const armem::query::data::Input& input, const _MemoryT& memory, const std::vector<query::data::QueryTarget>& ignoreTargets = {}) const
_MemoryT process(const armem::query::data::Input& input, const _MemoryT& memory, const std::vector<query::data::QueryTarget>& executeIf = {}) const
{
return this->process(input.memoryQueries, memory, ignoreTargets);
return this->process(input.memoryQueries, memory, executeIf);
}
void process(_MemoryT& result,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment