scrapeplanner/pyproject.toml at main · siddharthkumarx/scrapeplanner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
[build-system]
requires = ["hatchling>=1.27"]
build-backend = "hatchling.build"

[project]
name = "scrapeplanner"
dynamic = ["version"]
description = "Give it a URL; it reports the cheapest reliable, polite way to collect that site's data."
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
authors = [{ name = "Siddharth Kumar" }]
requires-python = ">=3.11"
keywords = ["web scraping", "robots.txt", "structured data", "json-ld", "sitemap", "crawling", "cli"]
classifiers = [
    "Development Status :: 4 - Beta",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Topic :: Internet :: WWW/HTTP",
    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
    "Typing :: Typed",
]
dependencies = [
    "httpx>=0.28,<1",
    "selectolax>=1.0,<2",
    "protego>=0.7",
    "typer>=0.27",
    "rich>=15",
    "pydantic>=2.14,<3",
    "pyyaml>=6.0.3",
]

[project.urls]
Homepage = "https://github.com/siddharthkumarx/scrapeplanner"
Issues = "https://github.com/siddharthkumarx/scrapeplanner/issues"
Changelog = "https://github.com/siddharthkumarx/scrapeplanner/blob/main/CHANGELOG.md"

[project.scripts]
scrapeplanner = "scrapeplanner.cli:main"

[dependency-groups]
dev = [
    "mypy>=2.4",
    "pytest>=9.2",
    "ruff>=0.17",
    "types-pyyaml>=6.0.12",
]
# Rendering and checking the README visuals (scripts/visuals, tests/visual).
visual = [
    "brotli>=1.1",
    "fonttools>=4.68",
    "markdown-it-py>=4.0",
    "pillow>=13.0",
    "playwright>=1.64",
]

[tool.hatch.version]
path = "src/scrapeplanner/__init__.py"

[tool.hatch.build.targets.sdist]
include = ["src/scrapeplanner", "LICENSE", "README.md", "CHANGELOG.md"]

[tool.hatch.build.targets.wheel]
packages = ["src/scrapeplanner"]

[tool.ruff]
line-length = 100
target-version = "py311"
extend-exclude = ["checks/results", "tests/fixtures"]

[tool.ruff.lint]
select = ["E", "W", "F", "I", "B", "UP", "SIM", "C4", "RUF", "PTH", "S", "N", "PL", "PT"]
ignore = [
    "PLR0911", "PLR0912", "PLR0913", "PLR0915", "PLR0917", "PLR2004",
    "N818",     # FetchError subclasses read better without an Error suffix
    "S105",     # "scrapeplanner" UA token is not a password
    "S101",     # assert in tests and type narrowing
    "S603",     # subprocess with trusted, list-form arguments
    "S607",     # "git" from PATH in repo tooling
]

[tool.ruff.lint.per-file-ignores]
"tests/**" = ["S", "PLC0415"]
"scripts/**" = ["T201"]
"checks/**" = ["T201"]

[tool.mypy]
strict = true
python_version = "3.11"
files = ["src", "tests", "scripts", "checks/run_checks.py"]
warn_unreachable = true

[[tool.mypy.overrides]]
module = ["fontTools.*"]
ignore_missing_imports = true

[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = ["-ra", "--strict-markers", "--strict-config", "-m", "not visual"]
markers = ["visual: renders the README visuals in Chromium (needs the 'visual' group)"]
xfail_strict = true