feat: introduce precompiled-grammars (#880)

.gitignore

@@ -21,3 +21,4 @@ cache
 .eslintcache
 report-engine-js-compat.json
 scripts/compares
+tm-grammars-themes

bench/bundle-test/bundle.bench.ts

@@ -0,0 +1,34 @@
+/* eslint-disable ts/ban-ts-comment */
+/* eslint-disable antfu/no-import-dist */
+
+import { bench, describe } from 'vitest'
+// @ts-ignore - ignore type error
+import { highlight as highlightA } from './dist/index-lite.min.mjs'
+// @ts-ignore - ignore type error
+import { highlight as highlightB } from './dist/index-wasm.min.mjs'
+
+const code = `
+import { ref } from 'vue'
+
+const message = ref('Hello World!')
+
+function reverseMessage() {
+  // Access/mutate the value of a ref via
+  // its .value property.
+  message.value = message.value.split('').reverse().join('')
+}
+
+function notify() {
+  alert('navigation was prevented.')
+}
+`
+
+describe('bundle', () => {
+  bench('js-precompiled', async () => {
+    await highlightA(code)
+  })
+
+  bench('wasm', async () => {
+    await highlightB(code)
+  })
+})

bench/bundle-test/index-lite.ts

@@ -0,0 +1,18 @@
+import { codeToHtml, createShikiInternal } from '@shikijs/core'
+import { createJavaScriptRawEngine } from '@shikijs/engine-javascript/raw'
+
+const shiki = createShikiInternal(
+  {
+    langs: [
+      import('@shikijs/langs-precompiled/ts'),
+    ],
+    themes: [
+      import('@shikijs/themes/vitesse-dark'),
+    ],
+    engine: createJavaScriptRawEngine(),
+  },
+)
+
+export async function highlight(code: string): Promise<string> {
+  return codeToHtml(await shiki, code, { lang: 'ts', theme: 'vitesse-dark' })
+}

bench/bundle-test/index-wasm.ts

@@ -0,0 +1,18 @@
+import { codeToHtml, createShikiInternal } from '@shikijs/core'
+import { createWasmOnigEngine } from '@shikijs/engine-oniguruma'
+
+const shiki = createShikiInternal(
+  {
+    langs: [
+      import('@shikijs/langs/ts'),
+    ],
+    themes: [
+      import('@shikijs/themes/vitesse-dark'),
+    ],
+    engine: createWasmOnigEngine(import('@shikijs/engine-oniguruma/wasm-inlined')),
+  },
+)
+
+export async function highlight(code: string): Promise<string> {
+  return codeToHtml(await shiki, code, { lang: 'ts', theme: 'vitesse-dark' })
+}

bench/bundle-test/package.json

@@ -0,0 +1,6 @@
+{
+  "private": true,
+  "scripts": {
+    "bench:prepare": "rollup -c && du -h dist/*"
+  }
+}

bench/bundle-test/rollup.config.mjs

@@ -0,0 +1,31 @@
+import resolve from '@rollup/plugin-node-resolve'
+import esbuild from 'rollup-plugin-esbuild'
+
+const plugins = [
+  resolve(),
+  esbuild({
+    minify: true,
+    target: 'esnext',
+  }),
+]
+
+export default [
+  {
+    input: 'index-wasm.ts',
+    output: {
+      file: 'dist/index-wasm.min.mjs',
+      format: 'es',
+      inlineDynamicImports: true,
+    },
+    plugins,
+  },
+  {
+    input: 'index-lite.ts',
+    output: {
+      file: 'dist/index-lite.min.mjs',
+      format: 'es',
+      inlineDynamicImports: true,
+    },
+    plugins,
+  },
+]

bench/engines.bench.ts → bench/engines/engines.bench.ts

@@ -1,20 +1,22 @@
 /* eslint-disable no-console */
 import type { BundledLanguage } from 'shiki'
-import type { ReportItem } from '../scripts/report-engine-js-compat'
+import type { ReportItem } from '../../scripts/report-engine-js-compat'
 import fs from 'node:fs/promises'
-import { createHighlighter, createJavaScriptRegexEngine, createOnigurumaEngine } from 'shiki'
+import { createJavaScriptRawEngine, createJavaScriptRegexEngine } from '@shikijs/engine-javascript'
+import { createHighlighter, createOnigurumaEngine } from 'shiki'
 import { bench, describe } from 'vitest'
 
 const js = createJavaScriptRegexEngine()
+const jsRaw = createJavaScriptRawEngine()
 const wasm = await createOnigurumaEngine(() => import('shiki/wasm'))
 
 const RANGE = [0, 20]
 
 // Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
-const report = await fs.readFile(new URL('../scripts/report-engine-js-compat.json', import.meta.url), 'utf-8').then(JSON.parse) as ReportItem[]
+const report = await fs.readFile(new URL('../../scripts/report-engine-js-compat.json', import.meta.url), 'utf-8').then(JSON.parse) as ReportItem[]
 const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang).slice(...RANGE) as BundledLanguage[]
 // Clone https://github.com/shikijs/textmate-grammars-themes to `../tm-grammars-themes`
-const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
+const samples = await Promise.all(langs.map(lang => fs.readFile(new URL(`../../tm-grammars-themes/samples/${lang}.sample`, import.meta.url), 'utf-8')))
 
 console.log('Benchmarking engines with', langs.length, 'languages')
 
@@ -30,14 +32,26 @@ const shikiWasm = await createHighlighter({
   engine: wasm,
 })
 
+const shikiJsPrecompiled = await createHighlighter({
+  langs: await Promise.all(langs.map(lang => import(`@shikijs/langs-precompiled/${lang}`))),
+  themes: ['vitesse-dark'],
+  engine: jsRaw,
+})
+
 for (const lang of langs) {
   describe(lang, () => {
+    const code = samples[langs.indexOf(lang)]
+
     bench('js', () => {
-      shikiJs.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
+      shikiJs.codeToTokensBase(code, { lang, theme: 'vitesse-dark' })
+    })
+
+    bench('js-precompiled', () => {
+      shikiJsPrecompiled.codeToTokensBase(code, { lang, theme: 'vitesse-dark' })
     })
 
     bench('wasm', () => {
-      shikiWasm.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
+      shikiWasm.codeToTokensBase(code, { lang, theme: 'vitesse-dark' })
     })
   })
 }

docs/package.json

@@ -12,14 +12,10 @@
   },
   "devDependencies": {
     "@iconify-json/svg-spinners": "catalog:",
-    "@shikijs/colorized-brackets": "workspace:*",
-    "@shikijs/transformers": "workspace:*",
-    "@shikijs/twoslash": "workspace:*",
     "@unocss/reset": "catalog:",
     "@vueuse/core": "catalog:",
     "floating-vue": "catalog:",
     "pinia": "catalog:",
-    "shiki": "workspace:*",
     "unocss": "catalog:",
     "unplugin-vue-components": "catalog:",
     "vitepress": "catalog:",

docs/packages/vitepress.md

@@ -39,9 +39,9 @@ export default defineConfig({
 And then in your [`.vitepress/theme/index.ts`](https://vitepress.dev/guide/custom-theme), install the Vue plugin and import the css with `@shikijs/vitepress-twoslash/styles.css`.
 
 ```ts twoslash
-import type { EnhanceAppContext } from 'vitepress' // [!code hl]
 // @noErrors: true
 // .vitepress/theme/index.ts
+import type { EnhanceAppContext } from 'vitepress' // [!code hl]
 import TwoslashFloatingVue from '@shikijs/vitepress-twoslash/client'
 import Theme from 'vitepress/theme'
 
@@ -86,7 +86,7 @@ console.log('hello')
 //      ^?
 ```
 
-<br> <!-- leaving some space for the query above -->
+<div class="h-100" /> <!-- leaving some space for the query above -->
 
 ### Vue Single File Component
 

package.json

@@ -13,7 +13,7 @@
     "docs": "pnpm -C docs run docs:dev",
     "docs:build": "pnpm -C docs run docs:build",
     "report-engine-js": "esno scripts/report-engine-js-compat.ts",
-    "bench": "vitest bench",
+    "bench": "pnpm -r run bench:prepare && vitest bench",
     "prepare": "simple-git-hooks"
   },
   "devDependencies": {
@@ -26,12 +26,15 @@
     "@rollup/plugin-node-resolve": "catalog:",
     "@rollup/plugin-replace": "catalog:",
     "@rollup/plugin-terser": "catalog:",
+    "@shikijs/colorized-brackets": "workspace:*",
     "@shikijs/engine-javascript": "workspace:*",
     "@shikijs/engine-oniguruma": "workspace:*",
     "@shikijs/markdown-it": "workspace:*",
     "@shikijs/monaco": "workspace:*",
     "@shikijs/rehype": "workspace:*",
     "@shikijs/transformers": "workspace:*",
+    "@shikijs/twoslash": "workspace:*",
+    "@shikijs/types": "workspace:*",
     "@shikijs/vitepress-twoslash": "workspace:*",
     "@types/fs-extra": "catalog:",
     "@types/hast": "catalog:",
@@ -62,7 +65,6 @@
     "rollup-plugin-copy": "catalog:",
     "rollup-plugin-dts": "catalog:",
     "rollup-plugin-esbuild": "catalog:",
-    "rollup-plugin-typescript2": "catalog:",
     "shiki": "workspace:*",
     "simple-git-hooks": "catalog:",
     "taze": "catalog:",
@@ -76,6 +78,7 @@
     "wrangler": "catalog:"
   },
   "resolutions": {
+    "@shikijs/colorized-brackets": "workspace:*",
     "@shikijs/compat": "workspace:*",
     "@shikijs/core": "workspace:*",
     "@shikijs/engine-javascript": "workspace:*",
@@ -84,6 +87,7 @@
     "@shikijs/rehype": "workspace:*",
     "@shikijs/transformers": "workspace:*",
     "@shikijs/twoslash": "workspace:*",
+    "@shikijs/types": "workspace:*",
     "@shikijs/vitepress-twoslash": "workspace:*",
     "@types/hast": "catalog:",
     "@types/mdast": "catalog:",

packages/compat/build.config.ts

@@ -8,6 +8,7 @@ export default defineBuildConfig({
   rollup: {
     emitCJS: false,
     dts: {
+      respectExternal: true,
       compilerOptions: {
         paths: {},
       },
@@ -16,5 +17,7 @@ export default defineBuildConfig({
   externals: [
     'hast',
     'shiki',
+    '@shikijs/types',
+    /^@shikijs[\\/].*/,
   ],
 })

packages/compat/package.json

@@ -38,7 +38,11 @@
   },
   "dependencies": {
     "@shikijs/core": "workspace:*",
+    "@shikijs/langs": "workspace:*",
+    "@shikijs/themes": "workspace:*",
     "@shikijs/transformers": "workspace:*",
+    "@shikijs/types": "workspace:*",
+    "@shikijs/vscode-textmate": "catalog:",
     "shiki": "workspace:*"
   },
   "devDependencies": {

packages/engine-javascript/build.config.ts

@@ -3,6 +3,8 @@ import { defineBuildConfig } from 'unbuild'
 export default defineBuildConfig({
   entries: [
     'src/index.ts',
+    'src/engine-compile.ts',
+    'src/engine-raw.ts',
   ],
   declaration: true,
   rollup: {