Skip to content
Snippets Groups Projects
Commit 06baf5f8 authored by Lorenz Gruber's avatar Lorenz Gruber
Browse files

insert components one by one to prevent database memory overflow

parent 5d23921f
No related branches found
No related tags found
No related merge requests found
Pipeline #66911 passed
...@@ -30,7 +30,7 @@ public: ...@@ -30,7 +30,7 @@ public:
this->desc["cfg-file"]=this->cfgFile.string(); this->desc["cfg-file"]=this->cfgFile.string();
} }
std::unordered_map<uint64_t,std::vector<std::string>> queryPathsForComponent(const std::vector<ComponentReference> & componentIds, const MemgraphConnection & memgraphConnection){ std::unordered_map<uint64_t,std::vector<std::string>> queryPathsForComponent(fishnet::util::forward_range_of<ComponentReference> auto & componentIds, const MemgraphConnection & memgraphConnection){
std::unordered_map<uint64_t,std::vector<std::string>> componentToFilesMap; std::unordered_map<uint64_t,std::vector<std::string>> componentToFilesMap;
std::vector<mg::Value> componentValues; std::vector<mg::Value> componentValues;
for(auto componentRef : componentIds) for(auto componentRef : componentIds)
...@@ -84,17 +84,10 @@ public: ...@@ -84,17 +84,10 @@ public:
} }
auto components = fishnet::graph::BFS::connectedComponents(graph).get(); auto components = fishnet::graph::BFS::connectedComponents(graph).get();
this->desc["Connected Components"]=components.size(); this->desc["Connected Components"]=components.size();
auto componentIds = memgraphClient.createComponents(components); auto componentIds = components
/** | std::views::transform([&memgraphClient](const auto & nodesOfComponent){return memgraphClient.createComponent(nodesOfComponent);})
* WITH $data as components UNWIND range(0,size(components)-1) as index | std::views::filter([](const std::optional<ComponentReference> & opt){return opt.has_value();})
CREATE (c:Component) WITH components[index] as nodes,c | std::views::transform([](const std::optional<ComponentReference> & opt){return opt.value();});
UNWIND nodes as nodeId
MATCH (n:Settlement) WHERE n.id=nodeId MERGE (n)-[:part_of]->(c)
Nodes: 6922201
UNWIND $data as component_id
MATCH (c:Component)<-[:part_of]-(:Settlement)-[:stored]->(f:File)WHERE ID(c)=component_id
*/
// Out of memory error seems to appear here:
std::unordered_map<uint64_t,std::vector<std::string>> componentToFilesMap = queryPathsForComponent(componentIds,memgraphClient.getMemgraphConnection()); std::unordered_map<uint64_t,std::vector<std::string>> componentToFilesMap = queryPathsForComponent(componentIds,memgraphClient.getMemgraphConnection());
std::unordered_map<std::string,std::vector<uint64_t>> fileToComponentsMap; std::unordered_map<std::string,std::vector<uint64_t>> fileToComponentsMap;
std::vector<ComponentFileJob> contractionJobs; std::vector<ComponentFileJob> contractionJobs;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment