From f4d284ad200fe2d0eee8516167673048e59666c0 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 18 Sep 2024 23:50:39 +0200 Subject: [PATCH 1/2] add robot.txt to `./web` --- web/package.json | 5 +++-- web/src/_build/gen-robots-txt.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 web/src/_build/gen-robots-txt.ts diff --git a/web/package.json b/web/package.json index eda38c4ab..93f039a96 100644 --- a/web/package.json +++ b/web/package.json @@ -61,11 +61,12 @@ "bundle:alone": "rsbuild build", "clean": "lerna run clean:alone --scope=@dzcode.io/web --include-dependencies --stream", "clean:alone": "rimraf dist coverage bundle node_modules/.cache && rimraf ./cloudflare/public", - "deploy": "npm run generate:htmls && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:prd ", - "deploy:stg": "npm run generate:htmls && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:stg", + "deploy": "npm run generate:htmls && npm run generate:robots-txt && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:prd ", + "deploy:stg": "npm run generate:htmls && npm run generate:robots-txt && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:stg", "e2e:dev": "npx wait-on http://localhost:8080/ && npx cypress open", "generate:bundle-info": "ts-node ../packages/tooling/bundle-info.ts", "generate:htmls": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/gen-multiple-htmls.ts", + "generate:robots-txt": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/gen-robots-txt.ts", "generate:sentry-release": "ts-node ../packages/tooling/sentry-release.ts web bundle", "generate:sitemap": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/sitemap.ts", "lint": "npm run build && npm run lint:alone", diff --git a/web/src/_build/gen-robots-txt.ts b/web/src/_build/gen-robots-txt.ts new file mode 100644 index 000000000..8874fcb76 --- /dev/null +++ b/web/src/_build/gen-robots-txt.ts @@ -0,0 +1,27 @@ +// for dev, run: +// npm run clean && npm run bundle && npm run generate:robots-txt +// npm run rsbuild preview + +import { join } from "path"; +import { writeFileSync } from "fs"; +import { Environment, environments } from "@dzcode.io/utils/dist/config/environment"; + +let stage = process.env.STAGE as Environment; +if (!environments.includes(stage)) { + console.log(`⚠️ No STAGE provided, falling back to "development"`); + stage = "development"; +} + +const distFolder = "./bundle"; + +const robotsTxtPath = join(distFolder, "robots.txt"); + +console.log(`generating $robot.txt ...`); + +const robotsTxt = `User-agent: * +${stage === "production" ? "Allow: /" : "Disallow: /"} +`; + +writeFileSync(robotsTxtPath, robotsTxt); + +console.log("done"); From 9ec7775cc110e36015539df788500562afb213c7 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 18 Sep 2024 23:51:14 +0200 Subject: [PATCH 2/2] fix robot.txt to `./api` --- api/package.json | 1 - api/src/app/index.ts | 5 +++-- api/src/app/middlewares/robots.ts | 24 ++++++++++++++++++------ package-lock.json | 10 ---------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/api/package.json b/api/package.json index 90dc7a1ac..ab56eaa95 100644 --- a/api/package.json +++ b/api/package.json @@ -22,7 +22,6 @@ "drizzle-orm": "^0.33.0", "express": "^4.21.0", "express-rate-limit": "^7.4.0", - "express-robots-txt": "^1.0.0", "fs-extra": "^11.2.0", "helmet": "^7.1.0", "lodash": "^4.17.21", diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 7424485d2..1d8c84623 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -20,7 +20,7 @@ import Container from "typedi"; import { ErrorMiddleware } from "./middlewares/error"; import { LoggerMiddleware } from "./middlewares/logger"; -import { RobotsMiddleware } from "./middlewares/robots"; +import { RobotsController } from "./middlewares/robots"; import { SecurityMiddleware } from "./middlewares/security"; // Use typedi container @@ -43,8 +43,9 @@ const routingControllersOptions: RoutingControllersOptions = { MilestoneController, ProjectController, ContributorController, + RobotsController, ], - middlewares: [SecurityMiddleware, ErrorMiddleware, LoggerMiddleware, RobotsMiddleware], + middlewares: [SecurityMiddleware, ErrorMiddleware, LoggerMiddleware], defaultErrorHandler: false, cors: Container.get(SecurityMiddleware).cors(), }; diff --git a/api/src/app/middlewares/robots.ts b/api/src/app/middlewares/robots.ts index b1d2c25e2..29caa869a 100644 --- a/api/src/app/middlewares/robots.ts +++ b/api/src/app/middlewares/robots.ts @@ -1,10 +1,22 @@ -import { RequestHandler } from "express"; -import { ExpressMiddlewareInterface, Middleware } from "routing-controllers"; +import { ConfigService } from "src/config/service"; import { Service } from "typedi"; -const robots = require("express-robots-txt"); // eslint-disable-line @typescript-eslint/no-require-imports +import { Response } from "express"; +import { Controller, Res, Get } from "routing-controllers"; @Service() -@Middleware({ type: "before", priority: 0 }) -export class RobotsMiddleware implements ExpressMiddlewareInterface { - use: RequestHandler = robots({ UserAgent: "*", Disallow: "/docs" }); +@Controller("/robots.txt") +export class RobotsController { + private readonly stage = this.configService.env().NODE_ENV; + + constructor(private readonly configService: ConfigService) {} + + @Get("/") + public async robotsTxt(@Res() response: Response): Promise { + response.set("Content-Type", "text/plain"); + if (this.stage === "production") { + return response.send("User-agent: *\nAllow: /"); + } else { + return response.send("User-agent: *\nDisallow: /"); + } + } } diff --git a/package-lock.json b/package-lock.json index e1b9f17e1..5288ab04e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -75,7 +75,6 @@ "drizzle-orm": "^0.33.0", "express": "^4.21.0", "express-rate-limit": "^7.4.0", - "express-robots-txt": "^1.0.0", "fs-extra": "^11.2.0", "helmet": "^7.1.0", "lodash": "^4.17.21", @@ -11287,15 +11286,6 @@ "express": "4 || 5 || ^5.0.0-beta.1" } }, - "node_modules/express-robots-txt": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/express-robots-txt/-/express-robots-txt-1.0.0.tgz", - "integrity": "sha512-ZzXTJgvHY6N8MIPzWidIrOGngkNJ8nrSP9Ox0nztVkQxvpwcX8egJvhM6tBUojf9xDmnSc7J5RlWQJJK91N3aA==", - "license": "MIT", - "peerDependencies": { - "express": "^4.12.1" - } - }, "node_modules/express-session": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.18.0.tgz",