1
0
mirror of https://github.com/go-task/task.git synced 2026-10-23 11:48:55 +03:00

feat(site): add versioned Algolia search and configuration export (#3031)

This commit is contained in:
Valentin Maerten
2026-09-19 12:07:30 +02:00
committed by GitHub
parent bd5d0cc865
commit 05c4708d87
10 changed files with 351 additions and 215 deletions

1
website/.gitignore vendored
View File

@@ -1,5 +1,6 @@
# Dependencies
/node_modules
/docsearch-export/
npm-debug.log*
yarn-debug.log*

View File

@@ -1 +1,2 @@
pnpm-lock.yaml
docsearch-export/

View File

@@ -1,6 +1,7 @@
import { defineConfig, HeadConfig } from 'vitepress';
import githubLinksPlugin from './plugins/github-links';
import { renderSearchContent } from './plugins/local-search';
import { searchKeywordsPlugin } from './plugins/search-keywords';
import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs';
import { resolve } from 'path';
import matter from 'gray-matter';
@@ -165,6 +166,12 @@ export default defineConfig({
{ name: 'docsearch:doc_type', content: pageData.frontmatter.docType }
]);
}
if (
pageData.frontmatter.search === false ||
pageData.frontmatter.noindex === true
) {
head.push(['meta', { name: 'docsearch:exclude', content: 'true' }]);
}
// Dynamic Open Graph and Twitter meta tags
const isHome = new URL(canonicalUrl).pathname === '/';
@@ -334,6 +341,7 @@ export default defineConfig({
});
md.use(tabsMarkdownPlugin);
md.use(groupIconMdPlugin);
md.use(searchKeywordsPlugin);
}
},
vite: {
@@ -383,13 +391,13 @@ export default defineConfig({
code: 'CESI65QJ',
placement: 'taskfiledev'
},
search: isProduction
search: isPublicDeploy
? {
provider: 'algolia',
options: {
appId: '7IZIJ13AI7',
apiKey: '34b64ae4fc8d9da43d9a13d9710aaddc',
indexName: 'taskfile'
indexName: isLatest ? 'taskfile' : 'taskfile-next'
}
}
: {
@@ -483,7 +491,7 @@ export default defineConfig({
}
},
sitemap: {
hostname: 'https://taskfile.dev'
hostname: isLatest ? 'https://taskfile.dev' : 'https://next.taskfile.dev'
},
buildEnd({ outDir }) {
const robots = isProduction
@@ -494,7 +502,21 @@ export default defineConfig({
'Sitemap: https://taskfile.dev/sitemap.xml',
''
]
: ['User-agent: *', 'Disallow: /', ''];
: isPublicDeploy
? [
'User-agent: Algolia Crawler',
'Allow: /docs/',
'Allow: /sitemap.xml',
'Allow: /$',
'Disallow: /',
'',
'User-agent: *',
'Disallow: /',
'',
'Sitemap: https://next.taskfile.dev/sitemap.xml',
''
]
: ['User-agent: *', 'Disallow: /', ''];
writeFileSync(resolve(outDir, 'robots.txt'), robots.join('\n'));
}
});

View File

@@ -8,6 +8,13 @@ export function renderSearchContent(
let html = md.render(src, env);
if (env.frontmatter?.search === false) return '';
// Only this index renderer adds the alternate terms to section content.
// The actual page keeps them in an attribute, invisible to readers.
html = html.replace(
/(<h[1-6]\b[^>]*\bdata-search-keywords="([^"]*)"[^>]*>[\s\S]*?<\/h[1-6]>)/g,
'$1<p>$2</p>'
);
// Otherwise stripping the tags joins the language label to the first
// command ("shellbrew"), preventing searches for "brew install".
html = html.replace(/<span class="lang">[^<]*<\/span>/g, '');

View File

@@ -0,0 +1,36 @@
import type MarkdownIt from 'markdown-it';
// Keep alternate search terms attached to the heading they describe. HTML
// attributes don't change the visible heading, its permalink or the outline.
export function searchKeywordsPlugin(md: MarkdownIt): void {
md.core.ruler.push('search-keywords', (state) => {
const keywords = state.env.frontmatter?.searchKeywords;
if (keywords === undefined) return;
const fail = (message: string): never => {
throw new Error(
`searchKeywords (${state.env.relativePath ?? 'page'}): ${message}`
);
};
if (!keywords || typeof keywords !== 'object' || Array.isArray(keywords)) {
fail('expected a mapping of heading IDs to lists of search terms');
}
const headings = new Map(
state.tokens
.filter((token) => token.type === 'heading_open')
.map((token) => [token.attrGet('id'), token])
);
for (const [anchor, terms] of Object.entries(keywords)) {
const heading = headings.get(anchor);
if (!heading) return fail(`heading #${anchor} does not exist`);
if (
!Array.isArray(terms) ||
!terms.length ||
terms.some((term) => typeof term !== 'string' || !term.trim())
) {
return fail(`#${anchor} needs a non-empty list of strings`);
}
heading.attrSet('data-search-keywords', terms.join(' '));
}
});
}

View File

@@ -1,16 +1,204 @@
// Algolia Crawler configuration for taskfile.dev.
// Run `pnpm docsearch:export` from website/ to generate the dashboard configuration
// and per-index settings in docsearch-export/. Preserve the crawler's write key
// when pasting the generated configuration. Apply changed settings to existing
// indices separately; initialIndexSettings only initializes new indices.
//
// This file is the source of truth. The crawler itself runs on Algolia's side,
// configured through the dashboard at https://crawler.algolia.com, and until now
// nothing described it here: `git log --all -- '*algolia*' '*docsearch*'`
// returned nothing, so the only copy lived in a web form nobody could review.
// When you change the configuration in the dashboard, change it here too.
//
// The API key below is the crawler's *write* key and is deliberately not in
// this repository. Keep the existing key when pasting this file into the
// dashboard; never replace this placeholder in Git.
//
// Every selector here was checked against the generated HTML, not assumed.
// Both actions share an extractor, but write to separate indices. Keep the
// extractor self-contained: the hosted crawler can serialize it independently.
const channels = [
{ origin: 'https://taskfile.dev', indexName: 'taskfile' },
{ origin: 'https://next.taskfile.dev', indexName: 'taskfile-next' }
];
const recordExtractor = ({ $, helpers, url }) => {
// ignoreNoIndex is needed for the next channel. Still honor page-level
// opt-outs and the released site's noindex directives.
if ($('meta[name="docsearch:exclude"]').attr('content') === 'true') return [];
if (
url.hostname === 'taskfile.dev' &&
/\b(noindex|none)\b/i.test($('meta[name="robots"]').attr('content') || '')
)
return [];
// The banner the llms plugin injects sits inside .vp-doc, ahead of the
// h1. It is display:none for readers and must not become content.
$('[data-nosnippet]').remove();
// DocSearch expects content selectors to target paragraphs or list
// items. Copy code blocks into crawler-only paragraphs so experienced
// users can search for exact Taskfile keys and command syntax without
// changing the page rendered to readers.
$('.vp-doc pre code').each((_, element) => {
const code = $(element).text().trim();
if (!code) return;
const paragraph = $('<p></p>').addClass('docsearch-code').text(code);
$(element).closest('pre').replaceWith(paragraph);
});
// Select each text subtree once. A list item may contain paragraphs and
// nested lists; selecting both parent and child repeats their content.
$('.vp-doc p, .vp-doc li, .vp-doc td, .vp-doc th').each((_, element) => {
if (!$(element).parents('.docsearch-content').length) {
$(element).addClass('docsearch-content');
}
});
// Frontmatter metadata is available after the refactor. Infer the same
// values from the URL while the old monolithic guide is still live, so
// this configuration can be installed before the website PR merges.
// Remove this URL inference once the refactored documentation is live
// and every indexed page exposes the DocSearch metadata.
const pathname = url.pathname.replace(/\/+$/, '') || '/';
const inferredSection = (() => {
if (pathname === '/docs') return 'Overview';
if (
/^\/docs\/(installation|getting-started|integrations)$/.test(pathname)
) {
return 'Getting Started';
}
if (/^\/docs\/reference\//.test(pathname)) return 'Reference';
if (/^\/docs\/(contributing|releasing|styleguide)$/.test(pathname)) {
return 'Contributing';
}
if (
/^\/docs\/(experiments|deprecations|security)(\/|$)/.test(pathname) ||
/^\/docs\/(changelog|faq|taskfile-versions|community)$/.test(pathname)
) {
return 'Project';
}
return 'Guide';
})();
const section =
$('meta[name="docsearch:section"]').attr('content') || inferredSection;
const docType =
$('meta[name="docsearch:doc_type"]').attr('content') ||
({
Overview: 'overview',
Reference: 'reference',
Contributing: 'contributing',
Project: 'project',
Guide: 'guide',
'Getting Started': 'guide'
}[section] ??
'guide');
const keywordsByAnchor = new Map();
$('.vp-doc [data-search-keywords]').each((_, element) => {
const heading = $(element);
keywordsByAnchor.set(
heading.attr('id'),
heading.attr('data-search-keywords')
);
});
const records = helpers.docsearch({
recordProps: {
// Not a heading on the page: the section the page belongs to,
// stated in its own frontmatter and emitted by transformHead. The
// usual DocSearch recipe reads the active sidebar link out of the
// DOM instead, which ties the index to the theme's markup and
// breaks silently when that markup changes.
lvl0: {
// Algolia documents an empty selector as the way to provide a
// raw, dynamically computed lvl0 through defaultValue.
selectors: '',
defaultValue: section
},
// Everything below is scoped to .vp-doc. VitePress renders the
// sidebar's section labels as <h2 class="text"> inside
// <aside class="VPSidebar">, five per page; an unscoped h2
// selector would index those on all 46 pages.
lvl1: '.vp-doc h1',
lvl2: '.vp-doc h2',
lvl3: '.vp-doc h3',
lvl4: '.vp-doc h4',
lvl5: '.vp-doc h5',
lvl6: '.vp-doc h6',
content: '.vp-doc .docsearch-content',
section: { defaultValue: section },
doc_type: { defaultValue: docType },
lang: {
defaultValue: $('html').attr('lang') || 'en-US'
},
// Deprecation notices remain findable, but don't outrank the page
// that documents the supported replacement.
pageRank: pathname.startsWith('/docs/deprecations/') ? '-10' : '0'
},
indexHeadings: true,
aggregateContent: true,
recordVersion: 'v3'
});
// Matching an alternate term should open its exact section. Adding
// page-wide keywords would make every section on the page compete.
return records.map((record) => ({
...record,
keywords: keywordsByAnchor.get(record.anchor) || ''
}));
};
const indexSettings = {
hitsPerPage: 20,
maxValuesPerFacet: 100,
attributesForFaceting: ['type', 'lang', 'section', 'doc_type'],
attributesToRetrieve: [
'hierarchy',
'content',
'anchor',
'url',
'url_without_anchor',
'type',
'lang',
'section',
'doc_type'
],
attributesToHighlight: ['hierarchy', 'content'],
attributesToSnippet: ['content:10'],
camelCaseAttributes: ['hierarchy', 'content'],
searchableAttributes: [
'unordered(hierarchy.lvl0)',
'unordered(hierarchy.lvl1)',
'unordered(hierarchy.lvl2)',
'unordered(hierarchy.lvl3)',
'unordered(hierarchy.lvl4)',
'unordered(hierarchy.lvl5)',
'unordered(hierarchy.lvl6)',
'unordered(keywords)',
'content'
],
distinct: true,
attributeForDistinct: 'url',
customRanking: [
'desc(weight.pageRank)',
'desc(weight.level)',
'asc(weight.position)'
],
ranking: [
'words',
'filters',
'typo',
'attribute',
'proximity',
'exact',
'custom'
],
highlightPreTag: '<span class="algolia-docsearch-suggestion--highlight">',
highlightPostTag: '</span>',
minWordSizefor1Typo: 3,
minWordSizefor2Typos: 7,
allowTyposOnNumericTokens: false,
minProximity: 1,
ignorePlurals: true,
advancedSyntax: true,
attributeCriteriaComputedByMinProximity: true,
removeWordsIfNoResults: 'allOptional',
separatorsToIndex: '_',
paginationLimitedTo: 1000,
exactOnSingleWordQuery: 'attribute',
queryType: 'prefixLast',
snippetEllipsisText: '',
alternativesAsExact: ['ignorePlurals', 'singleWordSynonym']
};
new Crawler({
appId: '7IZIJ13AI7',
@@ -19,208 +207,28 @@ new Crawler({
rateLimit: 8,
maxDepth: 10,
schedule: 'at 9:50 AM on Thursday',
// next retains latest canonicals and noindex for public search engines.
ignoreCanonicalTo: true,
ignoreNoIndex: true,
saveBackup: true,
safetyChecks: {
beforeIndexPublishing: {
maxLostRecordsPercentage: 10
},
// The dashboard's current config.d.ts exposes this at the safetyChecks
// level, rather than inside beforeIndexPublishing.
beforeIndexPublishing: { maxLostRecordsPercentage: 10 },
maxFailedUrls: 5
},
// Only the released site. next.taskfile.dev serves the same URLs from the
// upcoming release, and both sites share the single `taskfile` index, so
// crawling it as well would give every page a duplicate record.
// The root exists both before and after the documentation refactor. The old
// production site has no /docs/ landing page yet.
startUrls: ['https://taskfile.dev/'],
sitemaps: ['https://taskfile.dev/sitemap.xml'],
// Only /docs is indexed, so there is no reason to fetch the blog, the
// homepage or /adopters on every crawl.
discoveryPatterns: ['https://taskfile.dev/docs/**'],
exclusionPatterns: [
// Long, low-value for search, and it would outrank real pages on any
// version number or feature name it mentions.
'https://taskfile.dev/docs/changelog**',
// Scaffolding for writing new pages; already out of the sitemap.
'https://taskfile.dev/docs/*/template'
],
actions: [
{
indexName: 'taskfile',
pathsToMatch: ['https://taskfile.dev/docs/**'],
recordExtractor: ({ $, helpers, url }) => {
// The banner the llms plugin injects sits inside .vp-doc, ahead of the
// h1. It is display:none for readers and must not become content.
$('[data-nosnippet]').remove();
// DocSearch expects content selectors to target paragraphs or list
// items. Copy code blocks into crawler-only paragraphs so experienced
// users can search for exact Taskfile keys and command syntax without
// changing the page rendered to readers.
$('.vp-doc pre code').each((_, element) => {
const code = $(element).text().trim();
if (!code) return;
const paragraph = $('<p></p>').addClass('docsearch-code').text(code);
$(element).closest('pre').after(paragraph);
});
// Frontmatter metadata is available after the refactor. Infer the same
// values from the URL while the old monolithic guide is still live, so
// this configuration can be installed before the website PR merges.
// Remove this URL inference once the refactored documentation is live
// and every indexed page exposes the DocSearch metadata.
const pathname = url.pathname.replace(/\/+$/, '') || '/';
const inferredSection = (() => {
if (pathname === '/docs') return 'Overview';
if (
/^\/docs\/(installation|getting-started|integrations)$/.test(
pathname
)
) {
return 'Getting Started';
}
if (/^\/docs\/reference\//.test(pathname)) return 'Reference';
if (/^\/docs\/(contributing|releasing|styleguide)$/.test(pathname)) {
return 'Contributing';
}
if (
/^\/docs\/(experiments|deprecations|security)(\/|$)/.test(
pathname
) ||
/^\/docs\/(changelog|faq|taskfile-versions|community)$/.test(
pathname
)
) {
return 'Project';
}
return 'Guide';
})();
const section =
$('meta[name="docsearch:section"]').attr('content') ||
inferredSection;
const docType =
$('meta[name="docsearch:doc_type"]').attr('content') ||
({
Overview: 'overview',
Reference: 'reference',
Contributing: 'contributing',
Project: 'project',
Guide: 'guide',
'Getting Started': 'guide'
}[section] ??
'guide');
return helpers.docsearch({
recordProps: {
// Not a heading on the page: the section the page belongs to,
// stated in its own frontmatter and emitted by transformHead. The
// usual DocSearch recipe reads the active sidebar link out of the
// DOM instead, which ties the index to the theme's markup and
// breaks silently when that markup changes.
lvl0: {
// Algolia documents an empty selector as the way to provide a
// raw, dynamically computed lvl0 through defaultValue.
selectors: '',
defaultValue: section
},
// Everything below is scoped to .vp-doc. VitePress renders the
// sidebar's section labels as <h2 class="text"> inside
// <aside class="VPSidebar">, five per page; an unscoped h2
// selector would index those on all 46 pages.
lvl1: '.vp-doc h1',
lvl2: '.vp-doc h2',
lvl3: '.vp-doc h3',
lvl4: '.vp-doc h4',
lvl5: '.vp-doc h5',
lvl6: '.vp-doc h6',
content: '.vp-doc p, .vp-doc li, .vp-doc td, .vp-doc th',
section: { defaultValue: section },
doc_type: { defaultValue: docType },
lang: {
defaultValue: $('html').attr('lang') || 'en-US'
},
// Deprecation notices remain findable, but don't outrank the page
// that documents the supported replacement.
pageRank: pathname.startsWith('/docs/deprecations/') ? '-10' : '0'
},
indexHeadings: true,
aggregateContent: true,
recordVersion: 'v3'
});
}
}
],
// These settings initialize a new index. Algolia doesn't apply them to an
// existing index, so import the same values in the taskfile index settings
// when this configuration changes them.
initialIndexSettings: {
taskfile: {
hitsPerPage: 20,
maxValuesPerFacet: 100,
attributesForFaceting: ['type', 'lang', 'section', 'doc_type'],
attributesToRetrieve: [
'hierarchy',
'content',
'anchor',
'url',
'url_without_anchor',
'type',
'lang',
'section',
'doc_type'
],
attributesToHighlight: ['hierarchy', 'content'],
attributesToSnippet: ['content:10'],
camelCaseAttributes: ['hierarchy', 'content'],
searchableAttributes: [
'unordered(hierarchy.lvl0)',
'unordered(hierarchy.lvl1)',
'unordered(hierarchy.lvl2)',
'unordered(hierarchy.lvl3)',
'unordered(hierarchy.lvl4)',
'unordered(hierarchy.lvl5)',
'unordered(hierarchy.lvl6)',
'content'
],
distinct: true,
attributeForDistinct: 'url',
customRanking: [
'desc(weight.pageRank)',
'desc(weight.level)',
'asc(weight.position)'
],
ranking: [
'words',
'filters',
'typo',
'attribute',
'proximity',
'exact',
'custom'
],
highlightPreTag: '<span class="algolia-docsearch-suggestion--highlight">',
highlightPostTag: '</span>',
minWordSizefor1Typo: 3,
minWordSizefor2Typos: 7,
allowTyposOnNumericTokens: false,
minProximity: 1,
ignorePlurals: true,
advancedSyntax: true,
attributeCriteriaComputedByMinProximity: true,
removeWordsIfNoResults: 'allOptional',
separatorsToIndex: '_',
paginationLimitedTo: 1000,
exactOnSingleWordQuery: 'attribute',
queryType: 'prefixLast',
snippetEllipsisText: '',
alternativesAsExact: ['ignorePlurals', 'singleWordSynonym']
}
}
startUrls: channels.map(({ origin }) => `${origin}/`),
sitemaps: channels.map(({ origin }) => `${origin}/sitemap.xml`),
discoveryPatterns: channels.map(({ origin }) => `${origin}/docs/**`),
exclusionPatterns: channels.flatMap(({ origin }) => [
`${origin}/docs/changelog**`,
`${origin}/docs/**/template`,
`${origin}/docs/**/*.md`
]),
actions: channels.map(({ origin, indexName }) => ({
indexName,
pathsToMatch: [`${origin}/docs/**`],
recordExtractor
})),
initialIndexSettings: Object.fromEntries(
channels.map(({ indexName }) => [indexName, indexSettings])
)
});

View File

@@ -7,6 +7,7 @@
"dev": "vitepress dev",
"build": "vitepress build",
"check:links": "node scripts/check-markdown-links.mjs",
"docsearch:export": "node scripts/export-docsearch.mjs",
"preview": "vitepress preview",
"test": "node --test tests/*.test.mjs",
"lint": "prettier --write ."

View File

@@ -0,0 +1,56 @@
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { runInNewContext } from 'node:vm';
import { format } from 'prettier';
const websiteDir = fileURLToPath(new URL('../', import.meta.url));
const outDir = resolve(websiteDir, 'docsearch-export');
let config;
// Evaluate our local configuration without instantiating an Algolia client.
// Shared functions and settings are expanded into a standalone dashboard file.
runInNewContext(
await readFile(resolve(websiteDir, 'docsearch.config.js'), 'utf8'),
{
Crawler: function (value) {
config = value;
}
}
);
if (!config) throw new Error('docsearch.config.js did not define a Crawler');
function serialize(value) {
if (typeof value === 'function') return `(${value.toString()})`;
if (Array.isArray(value)) return `[${value.map(serialize).join(',')}]`;
if (value !== null && typeof value === 'object') {
return `{${Object.entries(value)
.map(([key, item]) => `${JSON.stringify(key)}:${serialize(item)}`)
.join(',')}}`;
}
return JSON.stringify(value);
}
await mkdir(outDir, { recursive: true });
async function write(name, content) {
const path = resolve(outDir, name);
await writeFile(path, content);
console.log(path);
}
await write(
'crawler.config.js',
await format(
'// Generated by pnpm docsearch:export. Edit docsearch.config.js instead.\n' +
'// Keep the existing write key when pasting into the Algolia dashboard.\n' +
`new Crawler(${serialize(config)});\n`,
{ parser: 'babel', singleQuote: true, trailingComma: 'none' }
)
);
for (const [name, settings] of Object.entries(config.initialIndexSettings)) {
await write(
`${name}.settings.json`,
JSON.stringify(settings, null, 2) + '\n'
);
}
console.log('Export only: no changes were sent to Algolia.');

View File

@@ -6,6 +6,8 @@ description:
section: Guide
docType: guide
outline: deep
searchKeywords:
task-aliases: [aliases, task aliases]
---
# Defining tasks

View File

@@ -6,6 +6,8 @@ description:
section: Guide
docType: guide
outline: deep
searchKeywords:
namespace-aliases: [aliases, namespace aliases]
---
# Including Taskfiles {#including-other-taskfiles}