Make the Loader also build the metadata db - 3b23306fee

Modified pom.xml

@@ -281,16 +281,17
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<configuration>
+ <workingDirectory>${project.basedir}</workingDirectory>
<executable>java</executable>
<arguments>
<argument>-classpath</argument>
<classpath/>
- <argument>-Xmx512m</argument>
- <argument>-Xms512m</argument>
+ <argument>-Xmx1024m</argument>
+ <argument>-Xms1024m</argument>
<argument>-Dlogback.configurationFile=src/test/resources/logback-test.xml</argument>
- <argument>-Dcerberus.recipes.filename=${project.basedir}/tmp/document.jsonlines</argument>
- <argument>-Dcerberus.recipes.datadir=${project.basedir}/tmp/hue</argument>
<argument>co.caio.cerberus.Loader</argument>
+ <argument>src/main/resources/application.properties</argument>
+ <argument>tmp/document.jsonlines</argument>
</arguments>
</configuration>
</plugin>

Modified src/main/java/co/caio/cerberus/Loader.java

@@ -1,34 +1,68
package co.caio.cerberus;

+import co.caio.cerberus.db.RecipeMetadata;
+import co.caio.cerberus.db.RecipeMetadataDatabase;
+import co.caio.cerberus.model.Recipe;
import co.caio.cerberus.search.IndexConfiguration;
import co.caio.cerberus.search.Indexer;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import reactor.core.publisher.Flux;

public class Loader {
private static final Logger logger = LoggerFactory.getLogger(Loader.class);

- public static void main(String[] args) throws Exception {
- var recipesFilename = System.getProperty("cerberus.recipes.filename");
- var dataDir = System.getProperty("cerberus.recipes.datadir");
- var serializer = new Serializer();
+ private final Path recipesFile;
+ private final Path dataDir;
+ private final Path dbDir;
+ private final int maxDbSize;
+ private final Serializer serializer = new Serializer();
+ private final String movePrefix;

- logger.info("Starting with recipes at {} and data at {}", recipesFilename, dataDir);
- assert dataDir != null && recipesFilename != null;
+ private Loader(Path configFile, Path inputFile) throws IOException {

+ assert configFile.toFile().isFile();
+ assert inputFile.toFile().isFile();
+
+ recipesFile = inputFile;
+
+ var conf = new Properties();
+ conf.load(Files.newBufferedReader(configFile));
+
+ dataDir = Path.of(conf.getProperty("cerberus.search.location"));
+ dbDir = Path.of(conf.getProperty("cerberus.search.lmdb-location"));
+ maxDbSize = Integer.parseInt(conf.getProperty("cerberus.search.lmdb-max-size"));
+
+ assert !dataDir.toFile().isFile() && !dbDir.toFile().isFile() && maxDbSize > 0;
+
+ movePrefix = "old." + System.currentTimeMillis() + ".";
+ }
+
+ private Stream<Recipe> recipeStream() throws IOException {
+ return Files.lines(recipesFile).map(serializer::readRecipe).flatMap(Optional::stream);
+ }
+
+ public void createIndex() throws IOException {
+
+ moveExistingDirs(dataDir);
+ dataDir.toFile().mkdirs();
+
+ logger.info("Initializing indexer");
var indexer =
new Indexer.Builder()
.analyzer(IndexConfiguration.getAnalyzer())
- .dataDirectory(Path.of(dataDir))
+ .dataDirectory(dataDir)
.createMode()
.build();
- logger.info("Initialized indexer", indexer);
- Files.lines(Path.of(recipesFilename))
- .map(serializer::readRecipe)
- .flatMap(Optional::stream)
+
+ logger.info("Ingesting all recipes. This will take a while...");
+ recipeStream()
.parallel()
.forEach(
recipe -> {

@@ -38,10 +72,63
logger.error("Exception adding recipe to index", e);
}
});
+
indexer.commit();
logger.info("Optimizing index for read-only usage");
indexer.mergeSegments();
- logger.info("Done!");
indexer.close();
+ logger.info("Finished creating lucene index");
+ }
+
+ public void createDatabase() throws IOException {
+
+ moveExistingDirs(dbDir);
+ dbDir.toFile().mkdirs();
+
+ logger.info("Creating metadata database at {}", dbDir);
+ var db = RecipeMetadataDatabase.Builder.open(dbDir, maxDbSize, false);
+
+ Flux.fromStream(recipeStream().map(RecipeMetadata::fromRecipe))
+ .buffer(100_000)
+ .subscribe(
+ recipes -> {
+ logger.info("Writing a batch of {} recipes", recipes.size());
+ db.saveAll(recipes);
+ });
+ logger.info("Finished creating metadata database");
+ }
+
+ private void moveExistingDirs(Path... paths) {
+ for (Path p : paths) {
+
+ if (!p.toFile().exists()) {
+ continue;
+ }
+
+ var newName = movePrefix + p.getFileName();
+ var dest = p.resolveSibling(newName);
+
+ if (p.toFile().renameTo(dest.toFile())) {
+ logger.info("Moved {} to {}", p, dest);
+ } else {
+ logger.error("Failed moving {} to {}", p, dest);
+ throw new RuntimeException("Unexpected error. Aborting...");
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ if (args.length != 2) {
+ logger.error("Required arguments: application.properties document.jsonlines");
+ System.exit(1);
+ }
+
+ var loader = new Loader(Path.of(args[0]), Path.of(args[1]));
+
+ loader.createDatabase();
+ loader.createIndex();
+
+ logger.info("Done!");
}
}

Deleted src/main/java/co/caio/cerberus/LMDBFlatBuffersExperiment.java

@@ -1,83 +1,0
-package co.caio.cerberus;
-
-import co.caio.cerberus.db.RecipeMetadata;
-import co.caio.cerberus.db.RecipeMetadataDatabase;
-import co.caio.cerberus.model.Recipe;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Optional;
-import java.util.stream.Stream;
-import reactor.core.publisher.Flux;
-
-public class LMDBFlatBuffersExperiment {
-
- private static final Serializer serializer = new Serializer();
- private static final String recipesFilename = "tmp/document.jsonlines";
- private static final int DB_SIZE_MB = 3_000;
-
- private static Stream<Recipe> recipeStream() throws Exception {
- return Files.lines(Path.of(recipesFilename))
- .map(serializer::readRecipe)
- .flatMap(Optional::stream);
- }
-
- private static void readDb(Path dbPath) throws Exception {
- var db = RecipeMetadataDatabase.Builder.open(dbPath, DB_SIZE_MB, true);
-
- recipeStream()
- .forEach(
- recipe -> {
- var recipeMetadata = db.findById(recipe.recipeId()).orElseThrow();
-
- if (recipeMetadata.getRecipeId() != recipe.recipeId()) {
- throw new RuntimeException(
- "Uh, reading recipe "
- + recipe.recipeId()
- + " gave me recipe "
- + recipeMetadata.getRecipeId());
- }
-
- if (!recipe.ingredients().equals(recipeMetadata.getIngredients())) {
- throw new RuntimeException(
- "Wanted "
- + recipe.ingredients()
- + " but got "
- + recipeMetadata.getIngredients());
- }
-
- if (!recipe.instructions().equals(recipeMetadata.getInstructions())) {
- throw new RuntimeException(
- "Wanted "
- + recipe.instructions()
- + " but got "
- + recipeMetadata.getInstructions());
- }
- });
- }
-
- private static void fancyWrite(Path dbPath) throws Exception {
- var db = RecipeMetadataDatabase.Builder.open(dbPath, DB_SIZE_MB, false);
- Flux.fromStream(recipeStream())
- .map(RecipeMetadata::fromRecipe)
- .buffer(250_000)
- .subscribe(
- batch -> {
- db.saveAll(batch);
- System.out.println("Wrote a batch of " + batch.size() + " recipes");
- });
- db.close();
- System.out.println("Done!");
- }
-
- private static void lmdbLoop(Path dbPath) throws Exception {
- System.out.println("Create DB at " + dbPath);
- fancyWrite(dbPath);
- System.out.println("Read DB from " + dbPath);
- readDb(dbPath);
- }
-
- public static void main(String[] args) throws Exception {
- var path = Path.of("tmp/lmdb-fancy-test");
- lmdbLoop(path);
- }
-}