DEICH-5952 disallow scraping in non-prod environments

deichman.no/server.js

@@ -181,7 +181,12 @@ app
     // Allow crawling all content
     server.use("/robots.txt", (req, res) => {
       res.type("text/plain");
-      res.send("User-agent: *\nDisallow:");
+
+      if (dev) {
+        res.send("User-agent: *\nDisallow: /"); // Disallow everything
+      } else {
+        res.send("User-agent: *\nDisallow:"); // Allow everything
+      }
     });
 
     // Check for redirects