Repository navigation
Expand file tree
/
Copy pathreport.py
More file actions
211 lines (191 loc) · 9.58 KB
/
Copy pathreport.py
File metadata and controls
211 lines (191 loc) · 9.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
import json
from dataclasses import asdict, dataclass
from datetime import date, datetime, timedelta
from pathlib import Path
from logger import get_logger
from registry import retrieve_dsfr_versions
logger = get_logger(__name__)
SUBTOOLS = {
"dom": ["dsfr", "rgaa", "gdpr", "tracking"],
"404": ["errors_404"],
}
@dataclass
class SummaryEntry:
dsfr: int | None = None
dsfr_enabled: bool = False
dsfr_version: str | None = None
ecoindex: int | None = None
ecoindex_size: float | None = None
ecoindex_requests: int | None = None
errors_404: int | None = None
errors_404_count: int | None = None
gdpr_cgu: int | None = None
gdpr_cgu_missing: list | None = None
gdpr_ml: int | None = None
gdpr_ml_missing: list | None = None
gdpr_pc: int | None = None
gdpr_pc_missing: list | None = None
lighthouse_accessibility: int | None = None
lighthouse_agentic_browsing: int | None = None
lighthouse_best_practices: int | None = None
lighthouse_performance: int | None = None
lighthouse_seo: int | None = None
observatory: int | None = None
observatory_failed: list | None = None
rgaa: int | None = None
rgaa_percentage: float | None = None
rgaa_version: str | None = None
rgaa_update_date: date | None = None
tracking: str | None = None
@dataclass
class ReportEntry:
def to_dict(self) -> dict:
result = asdict(self)
result["summary"] = {
(
"lighthouse_best-practices"
if key == "lighthouse_best_practices"
else "lighthouse_agentic-browsing"
if key == "lighthouse_agentic_browsing"
else key
): value
for key, value in result.get("summary", {}).items()
}
return result
url: str
date: datetime
summary: SummaryEntry
def get_dsfr_versions() -> dict:
if Path("data/dsfr_versions.json").exists():
with Path("data/dsfr_versions.json").open(encoding="utf-8") as file:
versions = json.load(file)
if date.fromisoformat(versions["fetched_at"]) >= date.today() - timedelta(days=30):
return versions["versions"]
logger.info("DSFR versions data not found or outdated, fetching latest versions...")
versions = retrieve_dsfr_versions()
with Path("data/dsfr_versions.json").open("w", encoding="utf-8") as file:
json.dump(versions, file, indent=2, ensure_ascii=False, default=str)
return versions["versions"]
def generate_report(data: dict, tools: list, verbose: bool = False) -> None:
dsfr_versions = get_dsfr_versions()
report = []
logger.info("Generating report for %s sites using tools: %s", len(data), ", ".join(tools))
for site, paths in data.items():
if verbose:
logger.info("Site: %s", site)
site_summary = ReportEntry(
url="https://" + site,
date=datetime.min,
summary=SummaryEntry(),
)
all_tools = set(tools)
for parent, subs in SUBTOOLS.items():
if parent in all_tools:
all_tools.remove(parent)
all_tools.update(subs)
found_tools = set()
for path in sorted(paths):
found_tools.add(path.stem)
if verbose:
logger.info(" [✓] %s", path.stem)
content = path.read_text(encoding="utf-8")
results = json.loads(content if content else "{}")
match path.stem:
case "errors_404":
total_links = len(results.get("links", []))
broken_links = len(results.get("broken", []))
score = (1 - broken_links / total_links) * 100 if total_links else 0
site_summary.summary.errors_404 = int(round(score, 0))
site_summary.summary.errors_404_count = broken_links
case "rgaa":
score = results.get("percentage")
site_summary.summary.rgaa_percentage = score
if score is None and results.get("mention"):
mention = results.get("mention", "").lower()
if "totalement conforme" in mention:
score = 100
elif "partiellement conforme" in mention:
score = 50
elif "non conforme" in mention:
score = 0
if not score:
score = 0
site_summary.summary.rgaa = int(round(score, 0))
site_summary.summary.rgaa_version = results.get("version")
site_summary.summary.rgaa_update_date = results.get("update_date")
case "dsfr":
score = 50 if results.get("header_brand") else 0
if f"v{results.get('version')}" in dsfr_versions:
index = list(dsfr_versions.keys()).index(f"v{results.get('version')}")
# the more recent the version, the higher the score (max 50 points)
# use a soft decay so recent versions keep a high score and slightly older versions are not penalized too harshly
# the curve should hit 0 at max index while keeping latest versions near the top
score += 50 * (1 - (min(index, len(dsfr_versions)) / len(dsfr_versions)) ** 0.8)
else:
# small score for using DSFR even if version is unknown or too old
score += 5 if results.get("version") else 0
site_summary.summary.dsfr = int(round(score, 0))
site_summary.summary.dsfr_enabled = bool(results.get("header_brand"))
site_summary.summary.dsfr_version = results.get("version")
case "ecoindex":
score = results.get("score", 0)
site_summary.summary.ecoindex = int(round(score, 0))
site_summary.summary.ecoindex_size = results.get("size")
site_summary.summary.ecoindex_requests = results.get("requests")
scan_date = results.get("date")
if scan_date:
date_obj = datetime.strptime(scan_date, "%Y-%m-%d %H:%M:%S.%f")
site_summary.date = max(date_obj.replace(microsecond=0), site_summary.date)
case "gdpr":
for key in ["ml", "pc", "cgu"]:
score = 50 if results.get(f"{key}_url") else 0
matches = results.get(f"{key}_matches", [])
missing = results.get(f"{key}_missing", [])
if matches or missing:
score += 50 * (len(matches) / (len(matches) + len(missing)))
setattr(site_summary.summary, f"gdpr_{key}", int(round(score, 0)))
setattr(
site_summary.summary,
f"gdpr_{key}_missing",
[x.split(" (ou) ", 1)[0] for x in missing],
)
case "lighthouse":
if results.get("runtimeError"):
logger.warning(" [⨯] Lighthouse runtime error for %s", site)
continue
for category in results.get("categories", []):
score = (results["categories"][category]["score"] or 0) * 100
setattr(site_summary.summary, f"lighthouse_{category.replace('-', '_')}", int(round(score, 0)))
scan_date = results.get("fetchTime")
if scan_date:
date_obj = datetime.strptime(scan_date, "%Y-%m-%dT%H:%M:%S.%fZ")
site_summary.date = max(date_obj.replace(microsecond=0), site_summary.date)
case "observatory":
score = results.get("scan", {}).get("score", 0)
score = (score / 125) * 100
site_summary.summary.observatory = int(round(score, 0))
site_summary.summary.observatory_failed = [
name for name, results in results.get("tests", {}).items() if not results.get("pass")
]
scan_date = results.get("scan", {}).get("responseHeaders", {}).get("date")
if scan_date:
try:
date_obj = datetime.strptime(scan_date, "%a, %d %b %Y %H:%M:%S %Z")
site_summary.date = max(date_obj.replace(microsecond=0), site_summary.date)
except ValueError:
logger.warning(" [⨯] Invalid date format: %s", scan_date)
case "tracking":
site_summary.summary.tracking = results.get("tools")[0] if results.get("tools") else None
scan_date = results.get("date")
if scan_date:
date_obj = datetime.strptime(scan_date, "%Y-%m-%d %H:%M:%S.%f")
site_summary.date = max(date_obj.replace(microsecond=0), site_summary.date)
case _:
continue
if verbose:
for tool in all_tools:
if tool not in found_tools:
logger.warning(" [⨯] %s (no data)", tool)
report.append(site_summary.to_dict())
with Path("data/report.json").open("w", encoding="utf-8") as report_file:
json.dump(report, report_file, indent=2, ensure_ascii=False, default=str)