remove embed pages and 404 from search results

website/src/lib/sitemap.ts

@@ -12,7 +12,7 @@ function generateSitemap() {
         const url = page.replace('.html', '').replace('index', '');
 
         const rootDir = url.split('/')[0];
-        if (languages[url] || languages[rootDir]) {
+        if (url.includes('embed') || url.includes('404') || languages[url] || languages[rootDir]) {
             // Skip localized pages
             return;
         }

website/static/robots.txt

@@ -1,3 +1,6 @@
 User-agent: *
 Disallow: /*?*
+Disallow: /embed
+Disallow: /*/embed
+Disallow: /404
 Sitemap: https://gpx.studio/sitemap.xml