feat(benchmark): Add benchmark for full language server

.github/workflows/benchmark.yaml

@@ -37,6 +37,9 @@ jobs:
       - name: Install dependencies
         run: just install
 
+      - name: Build (native)
+        run: just build release
+
       - name: Build (WASM)
         run: just build-wasm benchmark
 
@@ -46,5 +49,5 @@ jobs:
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@v3
         with:
-          run: cargo codspeed run && pnpm -C ./packages/benchmark-wasm run benchmark-codspeed
+          run: cargo codspeed run && pnpm -C ./packages/benchmark-wasm run benchmark-codspeed && pnpm -C ./packages/language-server-tests-benchmarks run benchmark-codspeed
           token: ${{ secrets.CODSPEED_TOKEN }}

justfile

@@ -33,3 +33,5 @@ benchmark:
 	cargo bench
 	echo "Running WASM benchmarks..."
 	pnpm -C ./packages/benchmark-wasm run benchmark --run
+	echo "Running Language Server benchmarks..."
+	pnpm -C ./packages/language-server-tests-benchmarks run benchmark --run

packages/language-server-tests-benchmarks/fixture/colors_benchmark.css

@@ -0,0 +1,17 @@
+.header {
+    background-color: #333;
+    color: white;
+    padding: 15px 20px;
+    text-align: center;
+}
+
+h1 {
+    color: red;
+    background-color: lab(50% 50% 50%);
+}
+
+.header .logo {
+    font-size: 2rem;
+    font-weight: bold;
+    color: lch(50% 50% 50%);
+}

packages/language-server-tests-benchmarks/fixture/folding_benchmark.css

@@ -0,0 +1,74 @@
+/* General Layout */
+.container {
+    width: 100%;
+    padding: 20px;
+    margin: 0 auto;
+    max-width: 1200px;
+}
+
+.header {
+    background-color: #333;
+    color: white;
+    padding: 15px 20px;
+    text-align: center;
+}
+
+.header .logo {
+    font-size: 2rem;
+    font-weight: bold;
+}
+
+.sidebar {
+    flex: 1;
+    padding: 20px;
+    background-color: #fff;
+    border-radius: 8px;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+.footer {
+    margin-top: 40px;
+    padding: 20px;
+    background-color: #222;
+    color: white;
+    text-align: center;
+    font-size: 0.9rem;
+}
+
+/* Media Queries */
+@media (max-width: 1024px) {
+    .content {
+        flex-direction: column;
+        align-items: center;
+    }
+    .main,
+    .sidebar {
+        width: 80%;
+        margin-bottom: 20px;
+    }
+}
+
+@media (max-width: 768px) {
+    .header .logo {
+        font-size: 1.5rem;
+    }
+    .footer {
+        font-size: 0.8rem;
+    }
+}
+
+@media (max-width: 480px) {
+    .container {
+        padding: 10px;
+    }
+    .header {
+        padding: 10px;
+    }
+    .content {
+        flex-direction: column;
+    }
+    .main,
+    .sidebar {
+        padding: 10px;
+    }
+}

packages/language-server-tests-benchmarks/fixture/hover_benchmark.css

@@ -0,0 +1,3 @@
+h1 {
+    color: red;
+}

packages/language-server-tests/package.json → packages/language-server-tests-benchmarks/package.json

@@ -1,13 +1,17 @@
 {
-	"name": "language-server-tests",
+	"name": "language-server-tests-benchmarks",
 	"type": "module",
 	"private": true,
 	"scripts": {
-		"test": "vitest"
+		"test": "vitest",
+		"benchmark": "vitest bench -c vitest.config.bench.ts",
+		"benchmark-codspeed": "CODSPEED=true pnpm run benchmark"
 	},
 	"dependencies": {
+		"@codspeed/vitest-plugin": "^3.1.1",
 		"@types/node": "^22.10.1",
 		"vitest": "^2.1.8",
+		"vscode-langservers-extracted": "^4.10.0",
 		"vscode-languageclient": "^9.0.1",
 		"vscode-languageserver-protocol": "^3.17.5",
 		"vscode-languageserver-textdocument": "^1.0.12",

packages/language-server-tests-benchmarks/src/benchmarks/css/colors.bench.ts

@@ -0,0 +1,54 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/colors_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+const weblspColors = await weblsp.sendDocumentColorRequest(weblspUri);
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+const vscodeColors = await vscodeLsp.sendDocumentColorRequest(vscodeLspUri);
+
+describe("Document Colors", async () => {
+	bench("weblsp - Document Colors", async () => {
+		await weblsp.sendDocumentColorRequest(weblspUri);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Document Colors", async () => {
+			await vscodeLsp.sendDocumentColorRequest(vscodeLspUri);
+		});
+	}
+});
+
+describe("Color Presentations", async () => {
+	bench("weblsp - Color Presentation", async () => {
+		await weblsp.sendColorPresentationRequest(
+			weblspUri,
+			weblspColors[0].color,
+			weblspColors[0].range
+		);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Color Presentation", async () => {
+			await vscodeLsp.sendColorPresentationRequest(
+				vscodeLspUri,
+				vscodeColors[0].color,
+				vscodeColors[0].range
+			);
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests-benchmarks/src/benchmarks/css/folding.bench.ts

@@ -0,0 +1,32 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/folding_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+
+describe("Folding Ranges", async () => {
+	bench("weblsp - Folding Ranges", async () => {
+		await weblsp.sendFoldingRangesRequest(weblspUri);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Folding Ranges", async () => {
+			await vscodeLsp.sendFoldingRangesRequest(vscodeLspUri);
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests-benchmarks/src/benchmarks/css/hover.bench.ts

@@ -0,0 +1,38 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/hover_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+
+describe("Hover", async () => {
+	bench("weblsp - Hover", async () => {
+		await weblsp.sendHoverRequest(weblspUri, {
+			line: 1,
+			character: 6,
+		});
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Hover", async () => {
+			await vscodeLsp.sendHoverRequest(vscodeLspUri, {
+				line: 1,
+				character: 6,
+			});
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests/tests/server.ts → packages/language-server-tests-benchmarks/src/server.ts

@@ -8,35 +8,61 @@ import { TextDocument } from "vscode-languageserver-textdocument";
 import { URI } from "vscode-uri";
 import * as assert from "node:assert/strict";
 import { fileURLToPath } from "node:url";
-import { createHash, randomBytes } from "node:crypto";
+import { randomBytes } from "node:crypto";
 
-const pathToBinary = fileURLToPath(
-	new URL("../../../target/debug/weblsp", import.meta.url)
-);
+let pathToBinary: string;
+if (process.env.BENCHMARK === "true" || process.env.RELEASE === "true") {
+	pathToBinary = fileURLToPath(
+		new URL("../../../target/release/weblsp", import.meta.url)
+	);
+} else {
+	pathToBinary = fileURLToPath(
+		new URL("../../../target/debug/weblsp", import.meta.url)
+	);
+}
 
 export const fixtureDir = URI.file(
 	fileURLToPath(new URL("./fixture", import.meta.url))
 ).toString();
 
-export type LanguageServerHandle = ReturnType<typeof startLanguageServer>;
+export type LanguageServerHandle = Awaited<
+	ReturnType<typeof startLanguageServer>
+>;
 
-export async function startLanguageServer(cwd?: string | undefined) {
-	console.info(`Starting language server at ${pathToBinary}`);
-	const childProcess = cp.spawn(pathToBinary, [], {
-		env: process.env,
-		cwd,
-		stdio: "pipe",
-	});
+export async function startLanguageServer(
+	cwd?: string | undefined,
+	which: "weblsp" | "vscode-css" = "weblsp"
+) {
+	if (which === "weblsp")
+		console.info(`Starting language server at ${pathToBinary}`);
+
+	const childProcess =
+		which === "weblsp"
+			? cp.spawn(pathToBinary, [], {
+					env: process.env,
+					cwd,
+					stdio: "pipe",
+			  })
+			: cp.fork(
+					"node_modules/vscode-langservers-extracted/bin/vscode-css-language-server",
+					["--stdio", `--clientProcessId=${process.pid.toString()}`],
+					{
+						execArgv: ["--nolazy"],
+						env: process.env,
+						cwd,
+						stdio: "pipe",
+					}
+			  );
 
 	if (!childProcess.stdout || !childProcess.stdin) {
 		throw new Error("Bad stdio configuration, should be pipe");
 	}
 
-	if (process.env.DEBUG) {
-		childProcess.stderr?.on("data", (data) => {
+	childProcess.stderr?.on("data", (data) => {
+		if (process.env.DEBUG) {
 			console.error(data.toString());
-		});
-	}
+		}
+	});
 
 	const connection = _.createProtocolConnection(
 		childProcess.stdout,
@@ -48,7 +74,7 @@ export async function startLanguageServer(cwd?: string | undefined) {
 	connection.onClose((e) => console.log("Closed", e));
 
 	connection.onUnhandledNotification((e) =>
-		console.log("Unhandled notificaiton", e)
+		console.log("Unhandled notification", e)
 	);
 
 	connection.onError((e) => console.log("Error:", e));
@@ -149,6 +175,15 @@ export async function startLanguageServer(cwd?: string | undefined) {
 		}
 	);
 
+	// VS Code's CSS language server crashes if this is not set
+	if (which === "vscode-css") {
+		Object.assign(settings, { "css.lint.validProperties": [] });
+		await connection.sendNotification(
+			_.DidChangeConfigurationNotification.type,
+			{ settings } satisfies _.DidChangeConfigurationParams
+		);
+	}
+
 	return {
 		process: childProcess,
 		connection,

packages/language-server-tests-benchmarks/src/tests/setup.ts

@@ -0,0 +1,9 @@
+import { startLanguageServer } from "../server";
+
+declare global {
+	var languageServer: import("../server").LanguageServerHandle;
+}
+
+if (!globalThis.languageServer) {
+	globalThis.languageServer = await startLanguageServer();
+}

packages/language-server-tests/tests/shutdown.test.ts → packages/language-server-tests-benchmarks/src/tests/shutdown.test.ts

@@ -1,5 +1,4 @@
 import { describe, expect, it } from "vitest";
-import { ServerCapabilities } from "vscode-languageserver-protocol/node";
 
 describe("Language server initilization", () => {
 	it("Can shutdown server", async () => {