Skip to content

Preserve time and timezone for regex-extracted dates (#174) #194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
apoorvdarshan wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Preserve time and timezone for regex-extracted dates (#174) #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 25 additions & 9 deletions htmldate/extractors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,16 @@

COMPLETE_URL = re.compile(rf"\D({YEAR_RE})[/_-]({MONTH_RE})[/_-]({DAY_RE})(?:\D|$)")

JSON_MODIFIED = re.compile(rf'"dateModified": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I)
JSON_PUBLISHED = re.compile(
rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I
# optional ISO-8601 time-of-day and time zone suffix (e.g. "T08:37:00+05:30")
TIME_TZ_RE = r"[T ][0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?"

JSON_MODIFIED = re.compile(
rf'"dateModified": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})?', re.I
)
TIMESTAMP_PATTERN = re.compile(
rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}"
JSON_PUBLISHED = re.compile(
rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})?', re.I
)
TIMESTAMP_PATTERN = re.compile(rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})")

# English, French, German, Indonesian and Turkish dates cache
MONTHS = [
Expand Down Expand Up @@ -444,19 +447,32 @@ def img_search(
return None


# strftime directives carrying time of day or time zone
TIME_TZ_DIRECTIVES = ("%H", "%I", "%M", "%S", "%f", "%z", "%Z", "%p", "%X", "%c")


def pattern_search(
text: str,
date_pattern: re.Pattern[str],
options: Extractor,
) -> str | None:
"Look for date expressions using a regular expression on a string of text."
match = date_pattern.search(text)
if match and is_valid_date(
if not match or not is_valid_date(
match[1], "%Y-%m-%d", earliest=options.min, latest=options.max
):
LOGGER.debug("regex found: %s %s", date_pattern, match[0])
return convert_date(match[1], "%Y-%m-%d", options.format)
return None
return None
LOGGER.debug("regex found: %s %s", date_pattern, match[0])
# carry time of day and time zone through when the output format needs them
# (regexes only capture the date part, the optional suffix holds time/tz)
if match.lastindex and match.lastindex >= 2 and match[2]:
if any(directive in options.format for directive in TIME_TZ_DIRECTIVES):
full = custom_parse(
match[1] + match[2], options.format, options.min, options.max
)
if full is not None:
return full
return convert_date(match[1], "%Y-%m-%d", options.format)


def json_search(
Expand Down
43 changes: 43 additions & 0 deletions tests/unit_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -741,6 +741,49 @@ def test_exact_date():
)


def test_free_text_timezone():
"""Time of day and time zone must be preserved when dates are extracted
from free text / JSON via regexes (issue #174)."""
tzformat = "%Y-%m-%d %H:%M:%S%z"

# JSON-LD datePublished with time and time zone
jsonld = (
'<html><head><script type="application/ld+json">'
'{"@context":"https://schema.org","@type":"Article",'
'"datePublished":"2024-11-06T08:37:00+05:30"}'
"</script></head><body><p>text</p></body></html>"
)
assert (
find_date(jsonld, outputformat=tzformat, original_date=True)
== "2024-11-06 08:37:00+0530"
)
# default date-only output stays unchanged
assert find_date(jsonld, original_date=True) == "2024-11-06"

# JSON-LD dateModified with time and time zone
jsonld_mod = (
'<html><head><script type="application/ld+json">'
'{"@context":"https://schema.org","@type":"Article",'
'"dateModified":"2024-11-06T08:37:00+05:30"}'
"</script></head><body><p>text</p></body></html>"
)
assert (
find_date(jsonld_mod, outputformat=tzformat, original_date=False)
== "2024-11-06 08:37:00+0530"
)

# timestamp found in free text of the body
freetext = (
"<html><body><p>Published on 2024-11-06T08:37:00+05:30 "
"by someone</p></body></html>"
)
assert (
find_date(freetext, outputformat=tzformat, original_date=True)
== "2024-11-06 08:37:00+0530"
)
assert find_date(freetext, original_date=True) == "2024-11-06"


def test_is_valid_date():
"""test internal date validation"""
assert (
Expand Down
Loading