-
Notifications
You must be signed in to change notification settings - Fork 40
Expand file tree
/
Copy pathlychee.toml
More file actions
155 lines (131 loc) · 4.91 KB
/
Copy pathlychee.toml
File metadata and controls
155 lines (131 loc) · 4.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# Lychee link checker configuration
# See: https://lychee.cli.rs/
# Accept these HTTP status codes as "OK"
accept = [200, 204, 301, 302]
# Root directory for resolving relative links
root_dir = "documentation/docs"
# Exclude SUMMARY.md (mdBook section headers) and .mise/ (shell scripts, not docs)
# Exclude Rust source files — lychee is only meant to check documentation;
# code comments may contain URL-like strings (e.g. multi-host connection strings)
# that are not real hyperlinks and would produce false-positive parse errors.
exclude_path = ["documentation/docs/SUMMARY.md", ".mise/", "crate"]
# Check links to files on disk
include_verbatim = false
# Maximum number of concurrent requests
max_concurrency = 16
# Number of retries for transient failures (5xx, timeouts, connection resets)
max_retries = 3
# Wait time between retries (seconds)
retry_wait_time = 2
# HTTP timeout (seconds)
timeout = 30
# User-Agent header
user_agent = "lychee/0.15 (Cosmian KMS docs; https://github.com/Cosmian/kms)"
# ── Excluded URL patterns ──────────────────────────────────────────────────
# Sites that block automated crawlers, rate-limit, or return 5xx to bots
exclude = [
# Internal French social security intranet URLs — not reachable from internet
'pkipubpc\.cnp\.recouv',
# Local / internal references
'localhost',
'127\.0\.0\.1',
'host\.docker\.internal',
'kms\.different\.com',
'@',
'eviden_vm',
'cbom',
'sbom',
'search/findex',
'\.\./cosmian_cli',
'\.\./cosmian_vm',
'\.\./search',
# URL schemes not supported by lychee
'chrome://',
'mailto:',
# NIST/IETF publication servers — rate-limit / occasionally return 5xx to bots
'csrc\.nist\.gov',
'nvlpubs\.nist\.gov',
'rfc-editor\.org',
# readthedocs.io — returns 429 to automated requests
'readthedocs\.io',
# Sites that block automated crawlers (valid in browser but return 4xx/5xx to bots)
# github.com consistently returns HTTP/2 protocol errors to automated crawlers/CI runners
'github\.com',
'cosmian\.com',
'package\.cosmian\.com',
'www\.mysql\.com',
# Percona docs refuses connections from automated link checkers
'docs\.percona\.com',
'crates\.io',
'support\.google\.com',
'admin\.google\.com',
'github\.com/sfackler',
'jwt\.io',
'manpages\.ubuntu\.com',
'webstore\.ansi\.org',
# ETSI — returns 503 to automated crawlers
'www\.etsi\.org',
# Placeholder/example URLs used in documentation
'vault\.azure\.net',
'kms\.my_domain\.com',
'kms\.different\.com',
'pkce_auth_example\.md',
'login\.microsoftonline\.com',
'googleapis\.com/oauth2',
'test_data/blob/main/configs/client/jwt\.toml',
# Non-routable IPs used in forward proxy tests
'1\.2\.3\.4',
# RFC-1918 private / link-local IPs used in SSRF regression tests (validate.rs)
# These are intentionally unreachable — they are test fixture URLs, not real links.
'10\.0\.0\.',
'172\.16\.0\.',
'192\.168\.',
'169\.254\.',
'metadata\.google\.internal',
'vault\.internal',
'kms\.svc\.cluster\.local',
# Sites that block automated crawlers or throttle bots
'percona\.com',
'percona\.github\.io',
# cargo-deny's default template placeholder registry example (deny.toml, commented out)
'sekretz\.com',
# Autogenerated CLI reference — not a real page
'kms_clients/cli/main_commands',
'kms_clients/installation',
# Miscellaneous sites that block or rate-limit bots
'knowledge\.workspace\.google\.com',
'dev\.mysql\.com',
'techdocs\.broadcom\.com',
'swagger\.io',
# IBM docs — returns 503 Service Unavailable (Guardium CM TDE guide)
'www\.ibm\.com',
# Eviden (Proteccio HSM vendor page) — returns 503 Service Unavailable to
# automated crawlers/CI runners
'eviden\.com',
# Kubernetes SIG documentation site — resets connections from automated crawlers
'secrets-store-csi-driver\.sigs\.k8s\.io',
# kubernetes.io — connection failures from CI runners
'kubernetes\.io',
# Google Workspace blog — aggressive 429 rate limiting from CI IPs
'workspaceupdates\.googleblog\.com',
# OVHcloud — blocks automated requests from CI runners
'ovhcloud\.com',
# InterSystems documentation — consistently times out from CI runners
'docs\.intersystems\.com',
# Ubuntu manpages — frequent timeouts from automated requests
'manpages\.ubuntu\.com',
# ANSSI (cyber.gouv.fr) — publications periodically restructure their URLs
'cyber\.gouv\.fr',
# Fortinet documentation — returns 503 to automated crawlers
'docs\.fortinet\.com',
# GNOME GitLab — returns 500 Internal Server Error to automated crawlers
'gitlab\.gnome\.org',
# Fragment/anchor patterns that are not real URLs
'get--export',
'not-possible',
'wrapped-by-an-hsm-key',
# Example/template registries in deny.toml — commented-out placeholder URL
'sekretz\.com',
# Local IP addresses used in test scripts
'127\.0\.0\.1',
]