diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java index 083988bd8..f6e095c83 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java @@ -77,10 +77,10 @@ public class FileSystem implements ModuleFileSystem, DBFileSystem { private MetaData metaData; @Override - public Path hostBase () { + public Path hostBase() { return hostBaseDirectory; } - + public ContentQuery query(final BiFunction nodeMapper) { return metaData.query(nodeMapper); } @@ -157,7 +157,7 @@ public List listDirectories(final Path base, final String start) { return listDirectories(folder); } - + public List listDirectories(final String folder) { List nodes = new ArrayList<>(); @@ -204,6 +204,7 @@ public List listContent(final Path base, final String start) { return listContent(folder); } + public List listContent(final String folder) { if ("".equals(folder)) { return metaData.listChildren(""); @@ -220,13 +221,13 @@ public List listSections(final Path contentFile) { return listSections(filename, folder); } - + public List listSections(final String filename, String folder) { List nodes = new ArrayList<>(); final Pattern isSectionOf = Constants.SECTION_OF_PATTERN.apply(filename); final Pattern isNamedSectionOf = Constants.SECTION_NAMED_OF_PATTERN.apply(filename); - + if ("".equals(folder)) { metaData.getTree().values() .stream() @@ -260,6 +261,10 @@ public List listSections(final String filename, String folder) { } private void addOrUpdateMetaData(Path file) { + addOrUpdateMetaData(file, false); + } + + private void addOrUpdateMetaData(Path file, boolean batch) { try { if (!Files.exists(file)) { return; @@ -283,10 +288,10 @@ private void addOrUpdateMetaData(Path file) { public void init() throws IOException { init(MetaData.Type.MEMORY); } - + public void init(MetaData.Type metaDataType) throws IOException { log.debug("init filesystem"); - + if (MetaData.Type.MEMORY.equals(metaDataType)) { this.metaData = new MemoryMetaData(); } else { @@ -331,7 +336,7 @@ public void onNext(FileEvent item) { reInitFolder(contentBase); fileWatcher.start(); - + eventBus.register(ReIndexContentMetaDataEvent.class, (event) -> { try { if (event.uri() == null) { @@ -356,37 +361,46 @@ private void swapMetaData() throws IOException { private void reInitFolder(final Path folder) throws IOException { - long before = System.currentTimeMillis(); - Files.walkFileTree(folder, new FileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - var uri = PathUtil.toRelativePath(dir, contentBase); - - metaData.createDirectory(uri); - return FileVisitResult.CONTINUE; - } + if (metaData instanceof PersistentMetaData pMetaData) { + pMetaData.startBatch(); + } + try { + long before = System.currentTimeMillis(); + Files.walkFileTree(folder, new FileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + var uri = PathUtil.toRelativePath(dir, contentBase); + + metaData.createDirectory(uri); + return FileVisitResult.CONTINUE; + } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - addOrUpdateMetaData(file); + addOrUpdateMetaData(file); - return FileVisitResult.CONTINUE; - } + return FileVisitResult.CONTINUE; + } - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - }); + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + }); - long after = System.currentTimeMillis(); + long after = System.currentTimeMillis(); - log.debug("loading metadata took " + (after - before) + "ms"); + log.debug("loading metadata took " + (after - before) + "ms"); + } finally { + if (metaData instanceof PersistentMetaData pMetaData) { + pMetaData.stopBatch(); + } + } } } diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/LuceneIndex.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/LuceneIndex.java index e39378647..b23e4da5a 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/LuceneIndex.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/LuceneIndex.java @@ -59,6 +59,12 @@ public class LuceneIndex implements AutoCloseable { private SearcherManager nrt_manager; private NRTCachingDirectory nrt_index; + private boolean batchMode = false; + + public void setBatchMode (boolean mode) { + batchMode = mode; + } + @Override public void close() throws Exception { if (nrt_manager != null) { @@ -78,17 +84,23 @@ public void commit() throws IOException { void add(Document document) throws IOException { writer.addDocument(document); - commit(); + if (!batchMode) { + commit(); + } } void update(Term term, Document document) throws IOException { writer.updateDocument(term, document); - commit(); + if (!batchMode) { + commit(); + } } void delete(Query query) throws IOException { writer.deleteDocuments(query); - commit(); + if (!batchMode) { + commit(); + } } List query(Query query, Sort sort) throws IOException { diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/PersistentMetaData.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/PersistentMetaData.java index aa1dd10c5..dea73230a 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/PersistentMetaData.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/metadata/persistent/PersistentMetaData.java @@ -87,6 +87,19 @@ public void close() throws IOException { } } + public void startBatch () { + index.setBatchMode(true); + } + + public void stopBatch () { + try { + index.setBatchMode(false); + index.commit(); + } catch (IOException ex) { + log.error("error commiting index", ex); + } + } + @Override public void addFile(String uri, Map data, LocalDate lastModified) {