mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-06 05:51:15 +08:00
docs: relocalize stale locale links (#61796)
* docs: relocalize stale locale links * docs: unify locale link postprocessing * docs: preserve relocalized frontmatter * docs: relocalize partial docs runs * docs: scope locale link postprocessing * docs: continue scoped relocalization * docs: drain parallel i18n results * docs: add i18n pipeline link regression tests * docs: clarify i18n pipeline regression test intent * docs: update provider references in i18n tests to use example-provider * fix: note docs i18n link relocalization * docs: rephrase gateway local ws sentence
This commit is contained in:
@@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Gateway/TUI: defer terminal chat finalization for per-attempt lifecycle errors so fallback retries keep streaming before the run is marked failed. (#60043) Thanks @jwchmodx.
|
||||
- TUI/command messages: strip inbound envelope metadata before rendering command/system messages so async completion notices stop leaking raw wrappers into the operator terminal. (#59985) Thanks @MoerAI.
|
||||
- TUI/terminal: restore Kitty keyboard protocol and `modifyOtherKeys` state on TUI exit and fatal CLI crashes so parent shells stop inheriting broken keyboard input after `openclaw tui` exits. (#49130) Thanks @biefan.
|
||||
- Docs/i18n: relocalize final localized-page links after translation so generated locale pages stop keeping stale English-root links when targets appear later in the same run. (#61796) thanks @hxy91819.
|
||||
|
||||
## 2026.4.5
|
||||
|
||||
|
||||
@@ -173,7 +173,7 @@ Fallback: SSH tunnel.
|
||||
ssh -N -L 18789:127.0.0.1:18789 user@host
|
||||
```
|
||||
|
||||
Then connect clients to `ws://127.0.0.1:18789` locally.
|
||||
Then connect clients locally to `ws://127.0.0.1:18789`.
|
||||
|
||||
<Warning>
|
||||
SSH tunnels do not bypass gateway auth. For shared-secret auth, clients still
|
||||
|
||||
@@ -17,15 +17,15 @@ const (
|
||||
bodyTagEnd = "</body>"
|
||||
)
|
||||
|
||||
func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool, routes *routeIndex) (bool, error) {
|
||||
func processFileDoc(ctx context.Context, translator docsTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, string, error) {
|
||||
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
currentHash := hashBytes(content)
|
||||
|
||||
@@ -33,10 +33,10 @@ func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, fil
|
||||
if !overwrite {
|
||||
skip, err := shouldSkipDoc(outputPath, currentHash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
if skip {
|
||||
return true, nil
|
||||
return true, "", nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, fil
|
||||
frontData := map[string]any{}
|
||||
if strings.TrimSpace(sourceFront) != "" {
|
||||
if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
|
||||
return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
}
|
||||
}
|
||||
frontTemplate, markers := buildFrontmatterTemplate(frontData)
|
||||
@@ -52,32 +52,30 @@ func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, fil
|
||||
|
||||
translatedDoc, err := translator.TranslateRaw(ctx, taggedInput, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
|
||||
return false, "", fmt.Errorf("translate failed (%s): %w", relPath, err)
|
||||
}
|
||||
|
||||
translatedFront, translatedBody, err := parseTaggedDocument(translatedDoc)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("tagged output invalid for %s: %w", relPath, err)
|
||||
return false, "", fmt.Errorf("tagged output invalid for %s: %w", relPath, err)
|
||||
}
|
||||
if sourceFront != "" && strings.TrimSpace(translatedFront) == "" {
|
||||
return false, fmt.Errorf("translation removed frontmatter for %s", relPath)
|
||||
return false, "", fmt.Errorf("translation removed frontmatter for %s", relPath)
|
||||
}
|
||||
if err := applyFrontmatterTranslations(frontData, markers, translatedFront); err != nil {
|
||||
return false, fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
|
||||
return false, "", fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
|
||||
}
|
||||
translatedBody = routes.localizeBodyLinks(translatedBody)
|
||||
|
||||
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
output := updatedFront + translatedBody
|
||||
return false, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
}
|
||||
|
||||
func formatTaggedDocument(frontMatter, body string) string {
|
||||
|
||||
@@ -19,7 +19,7 @@ type htmlReplacement struct {
|
||||
Value string
|
||||
}
|
||||
|
||||
func translateHTMLBlocks(ctx context.Context, translator *PiTranslator, body, srcLang, tgtLang string) (string, error) {
|
||||
func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, srcLang, tgtLang string) (string, error) {
|
||||
source := []byte(body)
|
||||
r := text.NewReader(source)
|
||||
md := goldmark.New(
|
||||
@@ -95,7 +95,7 @@ func sortHTMLReplacements(replacements []htmlReplacement) {
|
||||
})
|
||||
}
|
||||
|
||||
func translateHTMLBlock(ctx context.Context, translator *PiTranslator, htmlText, srcLang, tgtLang string) (string, error) {
|
||||
func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText, srcLang, tgtLang string) (string, error) {
|
||||
tokenizer := html.NewTokenizer(strings.NewReader(htmlText))
|
||||
var out strings.Builder
|
||||
skipDepth := 0
|
||||
|
||||
@@ -20,11 +20,24 @@ type docJob struct {
|
||||
type docResult struct {
|
||||
index int
|
||||
rel string
|
||||
output string
|
||||
duration time.Duration
|
||||
skipped bool
|
||||
err error
|
||||
}
|
||||
|
||||
type runConfig struct {
|
||||
targetLang string
|
||||
sourceLang string
|
||||
docsRoot string
|
||||
tmPath string
|
||||
mode string
|
||||
thinking string
|
||||
overwrite bool
|
||||
maxFiles int
|
||||
parallel int
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
|
||||
@@ -43,125 +56,159 @@ func main() {
|
||||
fatal(fmt.Errorf("no doc files provided"))
|
||||
}
|
||||
|
||||
resolvedDocsRoot, err := filepath.Abs(*docsRoot)
|
||||
if err != nil {
|
||||
if err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: *targetLang,
|
||||
sourceLang: *sourceLang,
|
||||
docsRoot: *docsRoot,
|
||||
tmPath: *tmPath,
|
||||
mode: *mode,
|
||||
thinking: *thinking,
|
||||
overwrite: *overwrite,
|
||||
maxFiles: *maxFiles,
|
||||
parallel: *parallel,
|
||||
}, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
|
||||
return NewPiTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
}); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if *tmPath == "" {
|
||||
*tmPath = filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("%s.tm.jsonl", *targetLang))
|
||||
func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslator docsTranslatorFactory) error {
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("no doc files provided")
|
||||
}
|
||||
|
||||
glossaryPath := filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("glossary.%s.json", *targetLang))
|
||||
resolvedDocsRoot, err := filepath.Abs(cfg.docsRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmPath := cfg.tmPath
|
||||
if tmPath == "" {
|
||||
tmPath = filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("%s.tm.jsonl", cfg.targetLang))
|
||||
}
|
||||
|
||||
glossaryPath := filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("glossary.%s.json", cfg.targetLang))
|
||||
glossary, err := LoadGlossary(glossaryPath)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
|
||||
routes, err := loadRouteIndex(resolvedDocsRoot, *targetLang)
|
||||
tm, err := LoadTranslationMemory(tmPath)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
|
||||
translator, err := NewPiTranslator(*sourceLang, *targetLang, glossary, *thinking)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
defer translator.Close()
|
||||
|
||||
tm, err := LoadTranslationMemory(*tmPath)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
|
||||
ordered, err := orderFiles(resolvedDocsRoot, files)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
totalFiles := len(ordered)
|
||||
preSkipped := 0
|
||||
if *mode == "doc" && !*overwrite {
|
||||
filtered, skipped, err := filterDocQueue(resolvedDocsRoot, *targetLang, ordered)
|
||||
if cfg.mode == "doc" && !cfg.overwrite {
|
||||
filtered, skipped, err := filterDocQueue(resolvedDocsRoot, cfg.targetLang, ordered)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
ordered = filtered
|
||||
preSkipped = skipped
|
||||
}
|
||||
if *maxFiles > 0 && *maxFiles < len(ordered) {
|
||||
ordered = ordered[:*maxFiles]
|
||||
if cfg.maxFiles > 0 && cfg.maxFiles < len(ordered) {
|
||||
ordered = ordered[:cfg.maxFiles]
|
||||
}
|
||||
|
||||
parallel := cfg.parallel
|
||||
if parallel < 1 {
|
||||
parallel = 1
|
||||
}
|
||||
|
||||
log.SetFlags(log.LstdFlags)
|
||||
start := time.Now()
|
||||
processed := 0
|
||||
skipped := 0
|
||||
localizedFiles := []string{}
|
||||
var runErr error
|
||||
|
||||
if *parallel < 1 {
|
||||
*parallel = 1
|
||||
}
|
||||
|
||||
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", *mode, totalFiles, len(ordered), preSkipped, *overwrite, *thinking, *parallel)
|
||||
switch *mode {
|
||||
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
|
||||
switch cfg.mode {
|
||||
case "doc":
|
||||
if *parallel > 1 {
|
||||
proc, skip, err := runDocParallel(context.Background(), ordered, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite, *parallel, glossary, *thinking, routes)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if parallel > 1 {
|
||||
proc, skip, outputs, err := runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, parallel, glossary, cfg.thinking, newTranslator)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
}
|
||||
} else {
|
||||
proc, skip, err := runDocSequential(context.Background(), ordered, translator, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite, routes)
|
||||
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
defer translator.Close()
|
||||
proc, skip, outputs, err := runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
}
|
||||
}
|
||||
case "segment":
|
||||
if *parallel > 1 {
|
||||
fatal(fmt.Errorf("parallel processing is only supported in doc mode"))
|
||||
if parallel > 1 {
|
||||
return fmt.Errorf("parallel processing is only supported in doc mode")
|
||||
}
|
||||
proc, err := runSegmentSequential(context.Background(), ordered, translator, tm, resolvedDocsRoot, *sourceLang, *targetLang, routes)
|
||||
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
return err
|
||||
}
|
||||
defer translator.Close()
|
||||
proc, outputs, err := runSegmentSequential(ctx, ordered, translator, tm, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang)
|
||||
processed += proc
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
}
|
||||
default:
|
||||
fatal(fmt.Errorf("unknown mode: %s", *mode))
|
||||
return fmt.Errorf("unknown mode: %s", cfg.mode)
|
||||
}
|
||||
|
||||
if err := tm.Save(); err != nil {
|
||||
fatal(err)
|
||||
if err := tm.Save(); err != nil && runErr == nil {
|
||||
runErr = err
|
||||
}
|
||||
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil && runErr == nil {
|
||||
runErr = err
|
||||
}
|
||||
elapsed := time.Since(start).Round(time.Millisecond)
|
||||
log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
|
||||
return runErr
|
||||
}
|
||||
|
||||
func runDocSequential(ctx context.Context, ordered []string, translator *PiTranslator, docsRoot, srcLang, tgtLang string, overwrite bool, routes *routeIndex) (int, int, error) {
|
||||
func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, []string, error) {
|
||||
processed := 0
|
||||
skipped := 0
|
||||
outputs := []string{}
|
||||
for index, file := range ordered {
|
||||
relPath := resolveRelPath(docsRoot, file)
|
||||
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
|
||||
start := time.Now()
|
||||
skip, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite, routes)
|
||||
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite)
|
||||
if err != nil {
|
||||
return processed, skipped, err
|
||||
return processed, skipped, outputs, err
|
||||
}
|
||||
if skip {
|
||||
skipped++
|
||||
log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
} else {
|
||||
processed++
|
||||
outputs = append(outputs, outputPath)
|
||||
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
return processed, skipped, nil
|
||||
return processed, skipped, outputs, nil
|
||||
}
|
||||
|
||||
func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite bool, parallel int, glossary []GlossaryEntry, thinking string, routes *routeIndex) (int, int, error) {
|
||||
func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite bool, parallel int, glossary []GlossaryEntry, thinking string, newTranslator docsTranslatorFactory) (int, int, []string, error) {
|
||||
jobs := make(chan docJob)
|
||||
results := make(chan docResult, len(ordered))
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
@@ -172,7 +219,7 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
translator, err := NewPiTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
translator, err := newTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
if err != nil {
|
||||
results <- docResult{err: err}
|
||||
return
|
||||
@@ -184,10 +231,11 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg
|
||||
}
|
||||
log.Printf("docs-i18n: [w%d %d/%d] start %s", workerID, job.index, len(ordered), job.rel)
|
||||
start := time.Now()
|
||||
skip, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite, routes)
|
||||
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite)
|
||||
results <- docResult{
|
||||
index: job.index,
|
||||
rel: job.rel,
|
||||
output: outputPath,
|
||||
duration: time.Since(start),
|
||||
skipped: skip,
|
||||
err: err,
|
||||
@@ -201,45 +249,58 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(jobs)
|
||||
for index, file := range ordered {
|
||||
jobs <- docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
|
||||
job := docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case jobs <- job:
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
processed := 0
|
||||
skipped := 0
|
||||
for i := 0; i < len(ordered); i++ {
|
||||
result := <-results
|
||||
if result.err != nil {
|
||||
wg.Wait()
|
||||
return processed, skipped, result.err
|
||||
outputs := []string{}
|
||||
var firstErr error
|
||||
for result := range results {
|
||||
if result.err != nil && firstErr == nil {
|
||||
firstErr = result.err
|
||||
}
|
||||
if result.skipped {
|
||||
skipped++
|
||||
log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
} else {
|
||||
} else if result.err == nil {
|
||||
processed++
|
||||
outputs = append(outputs, result.output)
|
||||
log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
return processed, skipped, nil
|
||||
return processed, skipped, outputs, firstErr
|
||||
}
|
||||
|
||||
func runSegmentSequential(ctx context.Context, ordered []string, translator *PiTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string, routes *routeIndex) (int, error) {
|
||||
func runSegmentSequential(ctx context.Context, ordered []string, translator docsTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string) (int, []string, error) {
|
||||
processed := 0
|
||||
outputs := []string{}
|
||||
for index, file := range ordered {
|
||||
relPath := resolveRelPath(docsRoot, file)
|
||||
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
|
||||
start := time.Now()
|
||||
if _, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang, routes); err != nil {
|
||||
return processed, err
|
||||
_, outputPath, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return processed, outputs, err
|
||||
}
|
||||
processed++
|
||||
outputs = append(outputs, outputPath)
|
||||
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
return processed, nil
|
||||
return processed, outputs, nil
|
||||
}
|
||||
|
||||
func resolveRelPath(docsRoot, file string) string {
|
||||
|
||||
76
scripts/docs-i18n/main_test.go
Normal file
76
scripts/docs-i18n/main_test.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeDocsTranslator struct{}
|
||||
|
||||
func (fakeDocsTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (fakeDocsTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
// Keep the fake translator deterministic so this test exercises the
|
||||
// docs-i18n pipeline wiring and final link relocalization, not model output.
|
||||
replaced := strings.NewReplacer(
|
||||
"Gateway", "网关",
|
||||
"See ", "参见 ",
|
||||
).Replace(text)
|
||||
return replaced, nil
|
||||
}
|
||||
|
||||
func (fakeDocsTranslator) Close() {}
|
||||
|
||||
func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
"",
|
||||
"See [Example provider](/providers/example-provider).",
|
||||
))
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "providers", "example-provider.md"), "# 示例 provider\n")
|
||||
|
||||
// This is the higher-level regression for the bug fixed in this PR:
|
||||
// if the pipeline stops wiring postprocess through the main flow, the final
|
||||
// localized output page will keep stale English-root links and this test fails.
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
parallel: 1,
|
||||
}, []string{filepath.Join(docsRoot, "gateway", "index.md")}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return fakeDocsTranslator{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
|
||||
expected := []string{
|
||||
"参见 [Troubleshooting](/zh-CN/gateway/troubleshooting).",
|
||||
"参见 [Example provider](/zh-CN/providers/example-provider).",
|
||||
}
|
||||
for _, want := range expected {
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected final localized page link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,37 +11,37 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string, routes *routeIndex) (bool, error) {
|
||||
func processFile(ctx context.Context, translator docsTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) (bool, string, error) {
|
||||
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
frontMatter, body := splitFrontMatter(string(content))
|
||||
frontData := map[string]any{}
|
||||
if frontMatter != "" {
|
||||
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
|
||||
return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := translateFrontMatter(ctx, translator, tm, frontData, relPath, srcLang, tgtLang); err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
body, err = translateHTMLBlocks(ctx, translator, body, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
segments, err := extractSegments(body, relPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
namespace := cacheNamespace()
|
||||
@@ -54,7 +54,7 @@ func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationM
|
||||
}
|
||||
translated, err := translator.Translate(ctx, seg.Text, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
|
||||
return false, "", fmt.Errorf("translate failed (%s): %w", relPath, err)
|
||||
}
|
||||
seg.Translated = translated
|
||||
entry := TMEntry{
|
||||
@@ -74,19 +74,18 @@ func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationM
|
||||
}
|
||||
|
||||
translatedBody := applyTranslations(body, segments)
|
||||
translatedBody = routes.localizeBodyLinks(translatedBody)
|
||||
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
return false, err
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
output := updatedFront + translatedBody
|
||||
return false, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
}
|
||||
|
||||
func splitFrontMatter(content string) (string, string) {
|
||||
@@ -134,7 +133,7 @@ func encodeFrontMatter(frontData map[string]any, relPath string, source []byte)
|
||||
return fmt.Sprintf("---\n%s---\n\n", string(encoded)), nil
|
||||
}
|
||||
|
||||
func translateFrontMatter(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, data map[string]any, relPath, srcLang, tgtLang string) error {
|
||||
func translateFrontMatter(ctx context.Context, translator docsTranslator, tm *TranslationMemory, data map[string]any, relPath, srcLang, tgtLang string) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -171,7 +170,7 @@ func translateFrontMatter(ctx context.Context, translator *PiTranslator, tm *Tra
|
||||
return nil
|
||||
}
|
||||
|
||||
func translateSnippet(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, segmentID, textValue, srcLang, tgtLang string) (string, error) {
|
||||
func translateSnippet(ctx context.Context, translator docsTranslator, tm *TranslationMemory, segmentID, textValue, srcLang, tgtLang string) (string, error) {
|
||||
if strings.TrimSpace(textValue) == "" {
|
||||
return textValue, nil
|
||||
}
|
||||
|
||||
40
scripts/docs-i18n/relocalize.go
Normal file
40
scripts/docs-i18n/relocalize.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func postprocessLocalizedDocs(docsRoot, targetLang string, localizedFiles []string) error {
|
||||
if targetLang == "" || targetLang == "en" || len(localizedFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
routes, err := loadRouteIndex(docsRoot, targetLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, path := range localizedFiles {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
frontMatter, body := splitFrontMatter(string(content))
|
||||
rewrittenBody := routes.localizeBodyLinks(body)
|
||||
if rewrittenBody == body {
|
||||
continue
|
||||
}
|
||||
|
||||
output := rewrittenBody
|
||||
if frontMatter != "" {
|
||||
output = "---\n" + frontMatter + "\n---\n\n" + rewrittenBody
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, []byte(output), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
220
scripts/docs-i18n/relocalize_test.go
Normal file
220
scripts/docs-i18n/relocalize_test.go
Normal file
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPostprocessLocalizedDocsFixesStaleLinksAfterLaterPagesExist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"), stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), stringsJoin(
|
||||
"---",
|
||||
"title: 故障排除",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"# 故障排除",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{
|
||||
filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"),
|
||||
filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"),
|
||||
}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
|
||||
if !strings.Contains(got, "---\ntitle: 网关\nx-i18n:\n source_hash: test\n---\n\n") {
|
||||
t.Fatalf("front matter corrupted after rewrite:\n%s", got)
|
||||
}
|
||||
want := "See [Troubleshooting](/zh-CN/gateway/troubleshooting)."
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected rewritten localized link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsRewritesPublishedPageLinksForEachLocale(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
lang string
|
||||
title string
|
||||
wantPrefix string
|
||||
}{
|
||||
{name: "zh-CN", lang: "zh-CN", title: "网关", wantPrefix: "/zh-CN"},
|
||||
{name: "ja-JP", lang: "ja-JP", title: "ゲートウェイ", wantPrefix: "/ja-JP"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
|
||||
writeFile(t, filepath.Join(docsRoot, tt.lang, "gateway", "troubleshooting.md"), "# Localized troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, tt.lang, "providers", "example-provider.md"), "# Localized example provider\n")
|
||||
|
||||
pagePath := filepath.Join(docsRoot, tt.lang, "gateway", "index.md")
|
||||
writeFile(t, pagePath, stringsJoin(
|
||||
"---",
|
||||
"title: "+tt.title,
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
"",
|
||||
"See [Example provider](/providers/example-provider).",
|
||||
"",
|
||||
`<Card href="/gateway/troubleshooting" title="Troubleshooting" />`,
|
||||
`<Card href="`+tt.wantPrefix+`/providers/example-provider" title="Example provider" />`,
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, tt.lang, []string{pagePath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, pagePath)
|
||||
expectedLinks := []string{
|
||||
"See [Troubleshooting](" + tt.wantPrefix + "/gateway/troubleshooting).",
|
||||
"See [Example provider](" + tt.wantPrefix + "/providers/example-provider).",
|
||||
`<Card href="` + tt.wantPrefix + `/gateway/troubleshooting" title="Troubleshooting" />`,
|
||||
`<Card href="` + tt.wantPrefix + `/providers/example-provider" title="Example provider" />`,
|
||||
}
|
||||
for _, want := range expectedLinks {
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected rewritten link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsOnlyTouchesScopedFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
|
||||
scopedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
unscopedPath := filepath.Join(docsRoot, "zh-CN", "help", "index.md")
|
||||
|
||||
writeFile(t, scopedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, unscopedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 帮助",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
beforeUnscoped := mustReadFile(t, unscopedPath)
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{scopedPath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
gotScoped := mustReadFile(t, scopedPath)
|
||||
if !containsLine(gotScoped, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
|
||||
t.Fatalf("expected scoped file rewrite, got:\n%s", gotScoped)
|
||||
}
|
||||
|
||||
afterUnscoped := mustReadFile(t, unscopedPath)
|
||||
if afterUnscoped != beforeUnscoped {
|
||||
t.Fatalf("expected unscoped file to remain unchanged\nbefore:\n%s\nafter:\n%s", beforeUnscoped, afterUnscoped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsContinuesAfterUnchangedFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
|
||||
unchangedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "already-localized.md")
|
||||
needsRewritePath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
|
||||
writeFile(t, unchangedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 已本地化",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/zh-CN/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, needsRewritePath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{unchangedPath, needsRewritePath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, needsRewritePath)
|
||||
if !containsLine(got, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
|
||||
t.Fatalf("expected later file rewrite after unchanged file, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func stringsJoin(lines ...string) string {
|
||||
result := ""
|
||||
for i, line := range lines {
|
||||
if i > 0 {
|
||||
result += "\n"
|
||||
}
|
||||
result += line
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read failed for %s: %v", path, err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func containsLine(text, want string) bool {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if line == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -22,6 +22,14 @@ type PiTranslator struct {
|
||||
client *docsPiClient
|
||||
}
|
||||
|
||||
type docsTranslator interface {
|
||||
Translate(context.Context, string, string, string) (string, error)
|
||||
TranslateRaw(context.Context, string, string, string) (string, error)
|
||||
Close()
|
||||
}
|
||||
|
||||
type docsTranslatorFactory func(string, string, []GlossaryEntry, string) (docsTranslator, error)
|
||||
|
||||
func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (*PiTranslator, error) {
|
||||
client, err := startDocsPiClient(context.Background(), docsPiClientOptions{
|
||||
SystemPrompt: translationPrompt(srcLang, tgtLang, glossary),
|
||||
|
||||
Reference in New Issue
Block a user