feat(db): use in memory db when MONGODB_URL not set (#1773)

* feat(db): allow use of in memory db when MONGODB_URL not set

* fix(db): enhance database disconnection handling with server stop

* feat(db): persist db to disk

* fix: use proper root folder in both build & dev

.dockerignore

@@ -8,4 +8,5 @@ node_modules/
 .svelte-kit/
 .env*
 !.env
-.env.local
+.env.local
+db

.gitignore

@@ -11,4 +11,5 @@ SECRET_CONFIG
 .idea
 !.env.ci
 !.env
-gcp-*.json
+gcp-*.json
+db

src/lib/jobs/refresh-assistants-counts.ts

@@ -26,49 +26,49 @@ async function refreshAssistantsCountsHelper() {
 	}
 
 	try {
-		await Database.getInstance()
-			.getClient()
-			.withSession((session) =>
-				session.withTransaction(async () => {
+		await (await Database.getInstance()).getClient().withSession((session) =>
+			session.withTransaction(async () => {
+				await (
 					await Database.getInstance()
-						.getCollections()
-						.assistants.aggregate([
-							{ $project: { _id: 1 } },
-							{ $set: { last24HoursCount: 0 } },
-							{
-								$unionWith: {
-									coll: "assistants.stats",
-									pipeline: [
-										{
-											$match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" },
+				)
+					.getCollections()
+					.assistants.aggregate([
+						{ $project: { _id: 1 } },
+						{ $set: { last24HoursCount: 0 } },
+						{
+							$unionWith: {
+								coll: "assistants.stats",
+								pipeline: [
+									{
+										$match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" },
+									},
+									{
+										$group: {
+											_id: "$assistantId",
+											last24HoursCount: { $sum: "$count" },
 										},
-										{
-											$group: {
-												_id: "$assistantId",
-												last24HoursCount: { $sum: "$count" },
-											},
-										},
-									],
-								},
+									},
+								],
 							},
-							{
-								$group: {
-									_id: "$_id",
-									last24HoursCount: { $sum: "$last24HoursCount" },
-								},
+						},
+						{
+							$group: {
+								_id: "$_id",
+								last24HoursCount: { $sum: "$last24HoursCount" },
 							},
-							{
-								$merge: {
-									into: "assistants",
-									on: "_id",
-									whenMatched: "merge",
-									whenNotMatched: "discard",
-								},
+						},
+						{
+							$merge: {
+								into: "assistants",
+								on: "_id",
+								whenMatched: "merge",
+								whenNotMatched: "discard",
 							},
-						])
-						.next();
-				})
-			);
+						},
+					])
+					.next();
+			})
+		);
 	} catch (e) {
 		logger.error(e, "Refresh assistants counter failed!");
 	}

src/lib/migrations/migrations.ts

@@ -13,7 +13,7 @@ export async function checkAndRunMigrations() {
 	}
 
 	// check if all migrations have already been run
-	const migrationResults = await Database.getInstance()
+	const migrationResults = await (await Database.getInstance())
 		.getCollections()
 		.migrationResults.find()
 		.toArray();
@@ -21,7 +21,7 @@ export async function checkAndRunMigrations() {
 	logger.info("[MIGRATIONS] Begin check...");
 
 	// connect to the database
-	const connectedClient = await Database.getInstance().getClient().connect();
+	const connectedClient = await (await Database.getInstance()).getClient().connect();
 
 	const lockId = await acquireLock(LOCK_KEY);
 
@@ -74,25 +74,23 @@ export async function checkAndRunMigrations() {
 				}. Applying...`
 			);
 
-			await Database.getInstance()
-				.getCollections()
-				.migrationResults.updateOne(
-					{ _id: migration._id },
-					{
-						$set: {
-							name: migration.name,
-							status: "ongoing",
-						},
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: "ongoing",
 					},
-					{ upsert: true }
-				);
+				},
+				{ upsert: true }
+			);
 
 			const session = connectedClient.startSession();
 			let result = false;
 
 			try {
 				await session.withTransaction(async () => {
-					result = await migration.up(Database.getInstance());
+					result = await migration.up(await Database.getInstance());
 				});
 			} catch (e) {
 				logger.info(`[MIGRATIONS]  "${migration.name}" failed!`);
@@ -101,18 +99,16 @@ export async function checkAndRunMigrations() {
 				await session.endSession();
 			}
 
-			await Database.getInstance()
-				.getCollections()
-				.migrationResults.updateOne(
-					{ _id: migration._id },
-					{
-						$set: {
-							name: migration.name,
-							status: result ? "success" : "failure",
-						},
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: result ? "success" : "failure",
 					},
-					{ upsert: true }
-				);
+				},
+				{ upsert: true }
+			);
 		}
 	}
 

src/lib/server/database.ts

@@ -14,29 +14,69 @@ import type { MigrationResult } from "$lib/types/MigrationResult";
 import type { Semaphore } from "$lib/types/Semaphore";
 import type { AssistantStats } from "$lib/types/AssistantStats";
 import type { CommunityToolDB } from "$lib/types/Tool";
-
+import { MongoMemoryServer } from "mongodb-memory-server";
 import { logger } from "$lib/server/logger";
 import { building } from "$app/environment";
 import type { TokenCache } from "$lib/types/TokenCache";
 import { onExit } from "./exitHandler";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { existsSync, mkdirSync } from "fs";
 
 export const CONVERSATION_STATS_COLLECTION = "conversations.stats";
 
+function findRepoRoot(startPath: string): string {
+	let currentPath = startPath;
+	while (currentPath !== "/") {
+		if (existsSync(join(currentPath, "package.json"))) {
+			return currentPath;
+		}
+		currentPath = dirname(currentPath);
+	}
+	throw new Error("Could not find repository root (no package.json found)");
+}
+
 export class Database {
-	private client: MongoClient;
+	private client?: MongoClient;
+	private mongoServer?: MongoMemoryServer;
 
 	private static instance: Database;
 
-	private constructor() {
+	private async init() {
 		if (!env.MONGODB_URL) {
-			throw new Error(
-				"Please specify the MONGODB_URL environment variable inside .env.local. Set it to mongodb://localhost:27017 if you are running MongoDB locally, or to a MongoDB Atlas free instance for example."
-			);
-		}
+			logger.warn("No MongoDB URL found, using in-memory server");
 
-		this.client = new MongoClient(env.MONGODB_URL, {
-			directConnection: env.MONGODB_DIRECT_CONNECTION === "true",
-		});
+			// Find repo root by looking for package.json
+			const currentFilePath = fileURLToPath(import.meta.url);
+			const repoRoot = findRepoRoot(dirname(currentFilePath));
+
+			// Use MONGO_STORAGE_PATH from env if set, otherwise use db/ in repo root
+			const dbPath = env.MONGO_STORAGE_PATH || join(repoRoot, "db");
+
+			logger.info(`Using database path: ${dbPath}`);
+			// Create db directory if it doesn't exist
+			if (!existsSync(dbPath)) {
+				logger.info(`Creating database directory at ${dbPath}`);
+				mkdirSync(dbPath, { recursive: true });
+			}
+
+			this.mongoServer = await MongoMemoryServer.create({
+				instance: {
+					dbName: env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : ""),
+					dbPath,
+				},
+				binary: {
+					version: "7.0.18",
+				},
+			});
+			this.client = new MongoClient(this.mongoServer.getUri(), {
+				directConnection: env.MONGODB_DIRECT_CONNECTION === "true",
+			});
+		} else {
+			this.client = new MongoClient(env.MONGODB_URL, {
+				directConnection: env.MONGODB_DIRECT_CONNECTION === "true",
+			});
+		}
 
 		this.client.connect().catch((err) => {
 			logger.error(err, "Connection error");
@@ -46,12 +86,17 @@ export class Database {
 		this.client.on("open", () => this.initDatabase());
 
 		// Disconnect DB on exit
-		onExit(() => this.client.close(true));
+		onExit(async () => {
+			logger.info("Closing database connection");
+			await this.client?.close(true);
+			await this.mongoServer?.stop();
+		});
 	}
 
-	public static getInstance(): Database {
+	public static async getInstance(): Promise<Database> {
 		if (!Database.instance) {
 			Database.instance = new Database();
+			await Database.instance.init();
 		}
 
 		return Database.instance;
@@ -61,6 +106,10 @@ export class Database {
 	 * Return mongoClient
 	 */
 	public getClient(): MongoClient {
+		if (!this.client) {
+			throw new Error("Database not initialized");
+		}
+
 		return this.client;
 	}
 
@@ -68,6 +117,10 @@ export class Database {
 	 * Return map of database's collections
 	 */
 	public getCollections() {
+		if (!this.client) {
+			throw new Error("Database not initialized");
+		}
+
 		const db = this.client.db(
 			env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "")
 		);
@@ -247,4 +300,4 @@ export class Database {
 
 export const collections = building
 	? ({} as unknown as ReturnType<typeof Database.prototype.getCollections>)
-	: Database.getInstance().getCollections();
+	: await Database.getInstance().then((db) => db.getCollections());

src/routes/assistants/+page.server.ts

@@ -1,6 +1,6 @@
 import { base } from "$app/paths";
 import { env } from "$env/dynamic/private";
-import { Database, collections } from "$lib/server/database.js";
+import { collections } from "$lib/server/database.js";
 import { SortKey, type Assistant } from "$lib/types/Assistant";
 import type { User } from "$lib/types/User";
 import { generateQueryTokens } from "$lib/utils/searchTokens.js";
@@ -58,9 +58,8 @@ export const load = async ({ url, locals }) => {
 		...shouldBeFeatured,
 	};
 
-	const assistants = await Database.getInstance()
-		.getCollections()
-		.assistants.find(filter)
+	const assistants = await collections.assistants
+		.find(filter)
 		.sort({
 			...(sort === SortKey.TRENDING && { last24HoursCount: -1 }),
 			userCount: -1,
@@ -70,9 +69,7 @@ export const load = async ({ url, locals }) => {
 		.limit(NUM_PER_PAGE)
 		.toArray();
 
-	const numTotalItems = await Database.getInstance()
-		.getCollections()
-		.assistants.countDocuments(filter);
+	const numTotalItems = await collections.assistants.countDocuments(filter);
 
 	return {
 		assistants: JSON.parse(JSON.stringify(assistants)) as Array<Assistant>,

src/routes/tools/+page.server.ts

@@ -1,6 +1,6 @@
 import { env } from "$env/dynamic/private";
 import { authCondition } from "$lib/server/auth.js";
-import { Database, collections } from "$lib/server/database.js";
+import { collections } from "$lib/server/database.js";
 import { toolFromConfigs } from "$lib/server/tools/index.js";
 import { SortKey } from "$lib/types/Assistant.js";
 import { ReviewStatus } from "$lib/types/Review";
@@ -60,9 +60,8 @@ export const load = async ({ url, locals }) => {
 		}),
 	};
 
-	const communityTools = await Database.getInstance()
-		.getCollections()
-		.tools.find(filter)
+	const communityTools = await collections.tools
+		.find(filter)
 		.skip(NUM_PER_PAGE * pageIndex)
 		.sort({
 			...(sort === SortKey.TRENDING && { last24HoursUseCount: -1 }),
@@ -84,9 +83,7 @@ export const load = async ({ url, locals }) => {
 
 	const tools = [...(pageIndex == 0 && !username ? configTools : []), ...communityTools];
 
-	const numTotalItems =
-		(await Database.getInstance().getCollections().tools.countDocuments(filter)) +
-		toolFromConfigs.length;
+	const numTotalItems = (await collections.tools.countDocuments(filter)) + toolFromConfigs.length;
 
 	return {
 		tools: JSON.parse(JSON.stringify(tools)) as CommunityToolDB[],