Merge pull request #71 from LD-Reborn/67-improve-fcp-by-defering-js-and-css

Added CriticalCSS, defered CSS and JS, fixed heading order, fixed fro…

.gitignore

@@ -18,3 +18,5 @@ src/Server/logs
 src/Shared/bin
 src/Shared/obj
 src/Server/wwwroot/logs/*
+src/Server/CriticalCSS/node_modules
+src/Server/CriticalCSS/package*.json

src/Server/Controllers/Frontend/HomeController.cs

@@ -20,8 +20,14 @@ public class HomeController : Controller
         _domainManager = domainManager;
     }
 
-    [Authorize]
     [HttpGet("/")]
+    public IActionResult Root()
+    {
+        return Redirect("/Home/Index");
+    }
+
+    [Authorize]
+    [HttpGet("Index")]
     public IActionResult Index()
     {
         return View();

src/Server/CriticalCSS/CriticalCSSGenerator.js

@@ -0,0 +1,129 @@
+import { generate } from 'critical';
+import fs from 'fs';
+import path from 'path';
+
+import puppeteer from 'puppeteer';
+
+const browser = await puppeteer.launch();
+const page = await browser.newPage();
+
+// Login
+await page.goto('http://localhost:5146/Account/Login');
+await page.type('#username', 'admin');
+await page.type('#password', 'UnsafePractice.67');
+await page.click('button[type=submit]');
+await page.waitForNavigation();
+
+// Extract cookies
+const cookies = await page.cookies();
+await browser.close();
+
+async function generateCriticalCSSForViews() {
+  const viewsDir = '../Views';
+  
+  // Helper function to get all .cshtml files recursively
+  function getAllCshtmlFiles(dir) {
+    let results = [];
+    const list = fs.readdirSync(dir);
+    
+    list.forEach(file => {
+      const filePath = path.join(dir, file);
+      const stat = fs.statSync(filePath);
+      console.log("DEBUG@2");
+      console.log(filePath);
+      if (stat && stat.isDirectory()) {
+        // Recursively get files from subdirectories
+        results = results.concat(getAllCshtmlFiles(filePath));
+      } else if (file.endsWith('.cshtml') && filePath.search("/_") == -1) {
+        results.push(filePath);
+      }
+    });
+    
+    return results;
+  }
+  
+  // Helper function to convert file path to URL path
+  function filePathToUrlPath(filePath) {
+    // Remove 'Views/' prefix
+    let relativePath = filePath.replace(/^Views[\/\\]/, '');
+    
+    // Remove .cshtml extension
+    relativePath = relativePath.replace(/\.cshtml$/, '');
+    
+    // Convert to URL format (replace \ with / and capitalize first letter)
+    const urlPath = relativePath
+      .split(/[\/\\]/)
+      .map((segment, index) => 
+        index === 0 ? segment : segment.charAt(0).toUpperCase() + segment.slice(1)
+      )
+      .join('/');
+    
+    // Handle the case where we have a single file (like Index.cshtml)
+    if (relativePath.includes('/')) {
+      // Convert to URL path format: Views/Home/Index.cshtml -> /Home/Index
+      return '/' + relativePath.replace(/\\/g, '/').replace(/\.cshtml$/, '');
+    } else {
+      // For files directly in Views folder (like Views/Index.cshtml)
+      return '/' + relativePath.replace(/\.cshtml$/, '');
+    }
+  }
+  
+  // Get all .cshtml files
+  const cshtmlFiles = getAllCshtmlFiles(viewsDir);
+  const criticalCssDir = '.';
+//   if (!fs.existsSync(criticalCssDir)) {
+//     fs.mkdirSync(criticalCssDir, { recursive: true });
+//   }
+  
+  // Process each file
+  for (const file of cshtmlFiles) {
+    try {
+      const urlPath = filePathToUrlPath(file).replace("../", "").replace("/Views", "");
+      
+      // Generate critical CSS
+      await generate({
+        src: `http://localhost:5146${urlPath}`,
+        inline: false,
+        width: 1920,
+        height: 1080,
+        penthouse: {
+          customHeaders: {
+            cookie: cookies.map(c => `${c.name}=${c.value}`).join('; ')
+          },
+          forceExclude: ['.btn'], // Otherwise buttons end up colorless and .btn overrides other classes like .btn-warning, etc. - so it has to be force-excluded here and re-added later
+          forceInclude: [
+            '[data-bs-theme=dark]',
+            '.navbar',
+            '.col-md-4',
+            '.visually-hidden', // visually hidden headings
+            '.bi-info-circle-fill', '.text-info', // info icon
+            '.container', '.col-md-6', '.row', '.g-4', '.row>*',
+            'p', '.fs-3', '.py-4', // title
+            '.mb-4',
+            '.card', '.card-body', '.p-2', // card
+            'h2', '.card-title', '.fs-5', // card - title
+            '.d-flex', '.justify-content-between', '.mt-2', // card - content
+            '.progress', '.mt-3', // card - progress bar
+            '.list-group', '.list-group-flush', '.list-group-item', '.list-group-flush>.list-group-item', '.list-group-flush>.list-group-item:last-child', '.badge', '.bg-warning', '.bg-success', '.h-100', // card - health check list
+            '.btn', '.btn-sm', '.btn-primary', '.btn-warning', '.btn-danger', // Searchdomains buttons
+            '.col-md-8', '.sidebar',
+            '.mb-0', '.mb-2', '.align-items-center',
+            'h3', '.col-md-3', '.col-md-2', '.text-nowrap', '.overflow-auto'
+          ]
+        },
+        target: {
+          css: path.join(criticalCssDir, urlPath.replace(/\//g, '.').replace(/^\./, '').replace("...", "") + '.css')
+        }
+      });
+      
+      console.log(`Critical CSS generated for: ${urlPath}`);
+    } catch (err) {
+      console.error(`Error processing ${file}:`, err);
+    }
+  }
+  
+  console.log('All critical CSS files generated!');
+}
+
+// Run the function
+generateCriticalCSSForViews().catch(console.error);

src/Server/CriticalCSS/README.md

@@ -0,0 +1,10 @@
+# How to use CriticalCSS
+1. Install it here
+```bash
+npm i -D critical
+npm install puppeteer
+```
+2. Run the css generator:
+```bash
+node CriticalCSSGenerator.js
+```

src/Server/Program.cs

@@ -13,6 +13,7 @@ using System.Reflection;
 using System.Configuration;
 using Microsoft.OpenApi.Models;
 using Shared.Models;
+using Microsoft.AspNetCore.ResponseCompression;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -115,6 +116,24 @@ builder.Services.AddAuthorization(options =>
         policy => policy.RequireRole("Admin"));
 });
 
+builder.Services.AddResponseCompression(options =>
+{
+    options.EnableForHttps = true;
+    options.Providers.Add<GzipCompressionProvider>();
+    options.Providers.Add<BrotliCompressionProvider>();
+    options.MimeTypes =
+    [
+        "text/plain",
+        "text/css",
+        "application/javascript",
+        "text/javascript",
+        "text/html",
+        "application/xml",
+        "text/xml",
+        "application/json",
+        "image/svg+xml"
+    ];
+});
 
 var app = builder.Build();
 
@@ -180,6 +199,8 @@ if (configuration.ApiKeys is not null)
     });
 }
 
+app.UseResponseCompression();
+
 // Add localization
 var supportedCultures = new[] { "de", "de-DE", "en-US" };
 var localizationOptions = new RequestLocalizationOptions()
@@ -189,6 +210,22 @@ var localizationOptions = new RequestLocalizationOptions()
 app.UseRequestLocalization(localizationOptions);
 
 app.MapControllers();
-app.UseStaticFiles();
+app.UseStaticFiles(new StaticFileOptions
+{
+    OnPrepareResponse = ctx =>
+    {
+        string requestPath = ctx.Context.Request.Path.ToString();
+        string[] cachedSuffixes = [".css", ".js", ".png", ".ico", ".woff2"];
+        if (cachedSuffixes.Any(suffix => requestPath.EndsWith(suffix)))
+        {
+            ctx.Context.Response.GetTypedHeaders().CacheControl =
+                new Microsoft.Net.Http.Headers.CacheControlHeaderValue()
+                {
+                    Public = true,
+                    MaxAge = TimeSpan.FromDays(365)
+                };
+        }
+    }
+});
 
 app.Run();

src/Server/Views/Home/Index.cshtml

@@ -18,9 +18,10 @@
 }
 
 <div class="container py-4">
-    <h3 class="mb-4">
+    <h1 class="visually-hidden">Searchdomains</h1>
+    <p class="mb-4 fs-3">
         @(hasName ? T["Hi, {0}!", name] : T["Hi!"])
-    </h3>
+    </p>
 
     <div class="row g-4">
 
@@ -28,7 +29,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Embedding Cache"]</h5>
+                    <h2 class="card-title fs-5">@T["Embedding Cache"]</h2>
 
                     <div class="d-flex justify-content-between">
                         <span>@T["Size"]</span>
@@ -62,7 +63,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Health Checks"]</h5>
+                    <h2 class="card-title fs-5">@T["Health Checks"]</h2>
 
                     <ul class="list-group list-group-flush">
                         <li class="list-group-item d-flex justify-content-between">
@@ -88,7 +89,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Searchdomains"]</h5>
+                    <h2 class="card-title fs-5">@T["Searchdomains"]</h2>
 
                     <div class="d-flex justify-content-between">
                         <span>@T["Count"]</span>
@@ -111,7 +112,7 @@
     </div>
 </div>
 
-<script defer>
+<script>
     var searchdomains = null;
 
     document.addEventListener('DOMContentLoaded', async () => {
@@ -138,41 +139,55 @@
         let healthchecksServer = document.getElementById("healthchecksServer");
         let healthchecksAiProvider = document.getElementById("healthchecksAiProvider");
 
-        listSearchdomains().then(async result => {
-            searchdomains = result.Searchdomains;
-            hideThrobber(searchdomainCount);
-            searchdomainCount.textContent = searchdomains.length;
+        (async() => {
+            listSearchdomains().then(async result => {
+                searchdomains = result.Searchdomains;
+                hideThrobber(searchdomainCount);
+                searchdomainCount.textContent = searchdomains.length;
 
-            var entityCount = 0;
-            var totalUtilization = 0;
-            for (var name in searchdomains){
-                let entityListResult = await listEntities(searchdomains[name]);
-                let entities = entityListResult.Results;
-                entityCount += entities.length;
-                let querycacheUtilizationResult = await getQuerycacheUtilization(searchdomains[name]);
-                let utilization = querycacheUtilizationResult.QueryCacheSizeBytes;
-                totalUtilization += utilization;
-            }
-            hideThrobber(searchdomainEntityCount);
-            hideThrobber(totalQuerycacheUtilization);
-            searchdomainEntityCount.textContent = entityCount;
-            totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(totalUtilization);
-        });
-        getEmbeddingcacheUtilization().then(result => {
-            let utilization = result.SizeInBytes;
-            let maxElementCount = result.MaxElementCount;
-            let elementCount = result.ElementCount;
-            let embeddingCount = result.EmbeddingsCount;
-            hideThrobber(embeddingcacheSize);
-            embeddingcacheSize.textContent = NumberOfBytesAsHumanReadable(utilization);
-            hideThrobber(embeddingcacheElementCount);
-            embeddingcacheElementCount.textContent = `${elementCount.toLocaleString()} / ${maxElementCount.toLocaleString()}`;
-            hideThrobber(embeddingcacheEmbeddingCount);
-            embeddingcacheEmbeddingCount.textContent = embeddingCount;
-            embeddingcacheElementCountProgressBar.style.width = `${elementCount / maxElementCount * 100}%`;
-        });
-        getHealthCheckStatusAndApply(healthchecksServer, "/healthz/Database");
-        getHealthCheckStatusAndApply(healthchecksAiProvider, "/healthz/AIProvider");
+                const perDomainPromises = searchdomains.map(async domain => {
+                    const [entityListResult, querycacheUtilizationResult] = await Promise.all([
+                        listEntities(domain),
+                        getQuerycacheUtilization(domain)
+                    ]);
+
+                    return {
+                        entityCount: entityListResult.Results.length,
+                        utilization: querycacheUtilizationResult.QueryCacheSizeBytes
+                    };
+                });
+
+                const results = await Promise.all(perDomainPromises);
+
+                let entityCount = 0;
+                let totalUtilization = 0;
+
+                for (const r of results) {
+                    entityCount += r.entityCount;
+                    totalUtilization += r.utilization;
+                }
+
+                hideThrobber(searchdomainEntityCount);
+                hideThrobber(totalQuerycacheUtilization);
+                searchdomainEntityCount.textContent = entityCount;
+                totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(totalUtilization);
+            });
+            getEmbeddingcacheUtilization().then(result => {
+                let utilization = result.SizeInBytes;
+                let maxElementCount = result.MaxElementCount;
+                let elementCount = result.ElementCount;
+                let embeddingCount = result.EmbeddingsCount;
+                hideThrobber(embeddingcacheSize);
+                embeddingcacheSize.textContent = NumberOfBytesAsHumanReadable(utilization);
+                hideThrobber(embeddingcacheElementCount);
+                embeddingcacheElementCount.textContent = `${elementCount.toLocaleString()} / ${maxElementCount.toLocaleString()}`;
+                hideThrobber(embeddingcacheEmbeddingCount);
+                embeddingcacheEmbeddingCount.textContent = embeddingCount;
+                embeddingcacheElementCountProgressBar.style.width = `${elementCount / maxElementCount * 100}%`;
+            });
+            getHealthCheckStatusAndApply(healthchecksServer, "/healthz/Database");
+            getHealthCheckStatusAndApply(healthchecksAiProvider, "/healthz/AIProvider");
+        })();
     });
 
 

src/Server/Views/Home/Searchdomains.cshtml

@@ -21,7 +21,7 @@
 }
 
 <div class="container-fluid mt-4">
-  <h1 class="visually-hidden">embeddingsearch</h1>
+  <h1 class="visually-hidden">Searchdomains</h1>
   <div class="row">
 
     <!-- Sidebar -->
@@ -804,7 +804,7 @@
             .getElementById('cacheClear')
             .addEventListener('click', () => {
                 const domainKey = getSelectedDomainKey();
-                fetch(`/Searchdomain/SearchCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
+                fetch(`/Searchdomain/QueryCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
                     method: 'POST'
                 }).then(response => {
                     if (response.ok) {
@@ -1031,7 +1031,7 @@
     }
 
     function getSearchdomainCacheUtilization(domainKey) {
-        return fetch(`/Searchdomain/SearchCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
+        return fetch(`/Searchdomain/QueryCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
             .then(r => r.json());
     }
 
@@ -1106,10 +1106,10 @@
             }
         });
         cacheUtilizationPromise.then(cacheUtilization => {
-            if (cacheUtilization != null && cacheUtilization.SearchCacheSizeBytes != null)
+            if (cacheUtilization != null && cacheUtilization.QueryCacheSizeBytes != null)
             {
                 document.querySelector('#cacheUtilization').innerText =
-                    `${NumberOfBytesAsHumanReadable(cacheUtilization.SearchCacheSizeBytes)}`;
+                    `${NumberOfBytesAsHumanReadable(cacheUtilization.QueryCacheSizeBytes)}`;
             } else {
                 showToast("@T["Unable to fetch searchdomain cache utilization"]", "danger");
                 console.error('Failed to fetch searchdomain cache utilization');

src/Server/Views/Shared/_Layout.cshtml

@@ -1,15 +1,39 @@
-@using Server.Services
+@using System.Globalization
+@using Server.Services
 @inject LocalizationService T
 
 <!DOCTYPE html>
-<html lang="en">
+<html lang="@CultureInfo.CurrentUICulture.TwoLetterISOLanguageName">
 <head>
     <meta charset="utf-8" />
+    <meta name="description" content="Embeddingsearch server" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@ViewData["Title"] - embeddingsearch</title>
-    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css">
-    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
+    @if (!Context.Request.Query.ContainsKey("renderRaw"))
+    {
+        <link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
+        <link rel="stylesheet" fetchpriority="high"
+        href="~/lib/bootstrap/dist/css/bootstrap.min.css"
+        media="print"
+        onload="this.media='all'">
+    }
+    <style>
+        @Html.Raw(File.ReadAllText(System.IO.Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "css", "site.css")))
+    </style>
+    @if (!Context.Request.Query.ContainsKey("noCriticalCSS"))
+    {
+        <style>
+            @if (Context.Request.Path.Value is not null)
+            {
+                string path = System.IO.Path.Combine("CriticalCSS", Context.Request.Path.Value.Trim('/').Replace("/", ".") + ".css");
+                Console.WriteLine(path);
+                if (File.Exists(path))
+                {
+                    @Html.Raw(File.ReadAllText(path));
+                }
+            }
+        </style>
+    }
     <script>
         window.appTranslations = {
             closeAlert: '@T["Close alert"]'
@@ -61,9 +85,9 @@
             &copy; 2025 - embeddingsearch
         </div>
     </footer>
-    <script src="~/lib/jquery/dist/jquery.min.js"></script>
-    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
-    <script src="~/js/site.js" asp-append-version="true"></script>
+    <script src="~/lib/jquery/dist/jquery.min.js" defer></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js" defer></script>
+    <script src="~/js/site.js" asp-append-version="true" defer></script>
     @await RenderSectionAsync("Scripts", required: false)
 </body>
 </html>

src/Server/wwwroot/css/site.css

@@ -49,3 +49,29 @@ body {
 .modal-title {
   font-size: 1.25rem;
 }
+
+/* Bootstrap icons */
+
+@font-face {
+  font-display: block;
+  font-family: "bootstrap-icons";
+  src: url("/fonts/bootstrap-icons.woff2") format("woff2"),
+url("/fonts/bootstrap-icons.woff") format("woff");
+}
+
+.bi::before,
+[class^="bi-"]::before,
+[class*=" bi-"]::before {
+  display: inline-block;
+  font-family: bootstrap-icons !important;
+  font-style: normal;
+  font-weight: normal !important;
+  font-variant: normal;
+  text-transform: none;
+  line-height: 1;
+  vertical-align: -.125em;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+.bi-info-circle-fill::before { content: "\f430"; }