From 97d9711372f47560525ba812f5a5d2d97dbcf094 Mon Sep 17 00:00:00 2001
From: creations <creations@creations.works>
Date: Mon, 12 May 2025 18:13:29 -0400
Subject: [PATCH] add robots txt route, verifiy required env vars func

---
 .env                  |  2 ++
 .gitignore            |  1 +
 config/environment.ts | 29 +++++++++++++++++++++++++++-
 src/index.ts          |  3 +++
 src/server.ts         | 45 +++++++++++++++++++++++++++++++------------
 5 files changed, 67 insertions(+), 13 deletions(-)

diff --git a/.env b/.env
index b069881..8a551d0 100644
--- a/.env
+++ b/.env
@@ -1,3 +1,5 @@
 # NODE_ENV=development
 HOST=0.0.0.0
 PORT=8080
+
+ROBOTS_FILE=
diff --git a/.gitignore b/.gitignore
index 58b5bae..e27b366 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /node_modules
 bun.lock
+robots.txt
diff --git a/config/environment.ts b/config/environment.ts
index cdd86e8..c203c7e 100644
--- a/config/environment.ts
+++ b/config/environment.ts
@@ -1,6 +1,33 @@
-export const environment: Environment = {
+import { resolve } from "node:path";
+import { logger } from "@creations.works/logger";
+
+const environment: Environment = {
 	port: Number.parseInt(process.env.PORT || "8080", 10),
 	host: process.env.HOST || "0.0.0.0",
 	development:
 		process.env.NODE_ENV === "development" || process.argv.includes("--dev"),
 };
+
+const robotstxtPath: string | null = process.env.ROBOTS_FILE
+	? resolve(process.env.ROBOTS_FILE)
+	: null;
+
+function verifyRequiredVariables(): void {
+	const requiredVariables = ["HOST", "PORT"];
+
+	let hasError = false;
+
+	for (const key of requiredVariables) {
+		const value = process.env[key];
+		if (value === undefined || value.trim() === "") {
+			logger.error(`Missing or empty environment variable: ${key}`);
+			hasError = true;
+		}
+	}
+
+	if (hasError) {
+		process.exit(1);
+	}
+}
+
+export { environment, robotstxtPath, verifyRequiredVariables };
diff --git a/src/index.ts b/src/index.ts
index c4148b4..6235bd6 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,8 +1,11 @@
 import { logger } from "@creations.works/logger";
 
 import { serverHandler } from "@/server";
+import { verifyRequiredVariables } from "@config/environment";
 
 async function main(): Promise<void> {
+	verifyRequiredVariables();
+
 	serverHandler.initialize();
 }
 
diff --git a/src/server.ts b/src/server.ts
index 5646636..066b2ef 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -1,5 +1,5 @@
 import { resolve } from "node:path";
-import { environment } from "@config/environment";
+import { environment, robotstxtPath } from "@config/environment";
 import { logger } from "@creations.works/logger";
 import {
 	type BunFile,
@@ -93,7 +93,39 @@ class ServerHandler {
 		const extendedRequest: ExtendedRequest = request as ExtendedRequest;
 		extendedRequest.startPerf = performance.now();
 
+		const headers = request.headers;
+		let ip = server.requestIP(request)?.address;
+
+		if (!ip || ip.startsWith("172.") || ip === "127.0.0.1") {
+			ip =
+				headers.get("CF-Connecting-IP")?.trim() ||
+				headers.get("X-Real-IP")?.trim() ||
+				headers.get("X-Forwarded-For")?.split(",")[0].trim() ||
+				"unknown";
+		}
+
 		const pathname: string = new URL(request.url).pathname;
+		if (pathname === "/robots.txt" && robotstxtPath) {
+			try {
+				const file: BunFile = Bun.file(robotstxtPath);
+				if (await file.exists()) {
+					const fileContent: ArrayBuffer = await file.arrayBuffer();
+					const contentType: string = file.type || "text/plain";
+					return new Response(fileContent, {
+						headers: { "Content-Type": contentType },
+					});
+				}
+				logger.warn(`File not found: ${robotstxtPath}`);
+				return new Response("Not Found", { status: 404 });
+			} catch (error) {
+				logger.error([
+					`Error serving robots.txt: ${robotstxtPath}`,
+					error as Error,
+				]);
+				return new Response("Internal Server Error", { status: 500 });
+			}
+		}
+
 		if (pathname.startsWith("/public") || pathname === "/favicon.ico") {
 			return await this.serveStaticFile(pathname);
 		}
@@ -220,17 +252,6 @@ class ServerHandler {
 			);
 		}
 
-		const headers = request.headers;
-		let ip = server.requestIP(request)?.address;
-
-		if (!ip || ip.startsWith("172.") || ip === "127.0.0.1") {
-			ip =
-				headers.get("CF-Connecting-IP")?.trim() ||
-				headers.get("X-Real-IP")?.trim() ||
-				headers.get("X-Forwarded-For")?.split(",")[0].trim() ||
-				"unknown";
-		}
-
 		logger.custom(
 			`[${request.method}]`,
 			`(${response.status})`,