From db214871b0123411c2042a8b1c1178fbb6622a61 Mon Sep 17 00:00:00 2001 From: Apoorv Darshan Date: Thu, 9 Jul 2026 22:29:51 +0530 Subject: [PATCH] Preserve time and timezone for regex-extracted dates (#174) Dates extracted from free text and JSON-LD go through regex patterns (JSON_PUBLISHED, JSON_MODIFIED, TIMESTAMP_PATTERN) that only captured the YYYY-MM-DD part, so find_date dropped the time of day and timezone even when the requested outputformat asked for them (e.g. '%Y-%m-%d %H:%M:%S%z'). Capture the optional ISO-8601 time/timezone suffix in these patterns and, when the output format contains time or timezone directives, parse the full timestamp through custom_parse (the same routine the HTML-markup path already uses). Date-only output is unchanged. --- htmldate/extractors.py | 34 ++++++++++++++++++++++++--------- tests/unit_tests.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/htmldate/extractors.py b/htmldate/extractors.py index bd0c470c..8129eb25 100644 --- a/htmldate/extractors.py +++ b/htmldate/extractors.py @@ -122,13 +122,16 @@ COMPLETE_URL = re.compile(rf"\D({YEAR_RE})[/_-]({MONTH_RE})[/_-]({DAY_RE})(?:\D|$)") -JSON_MODIFIED = re.compile(rf'"dateModified": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I) -JSON_PUBLISHED = re.compile( - rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I +# optional ISO-8601 time-of-day and time zone suffix (e.g. "T08:37:00+05:30") +TIME_TZ_RE = r"[T ][0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?" + +JSON_MODIFIED = re.compile( + rf'"dateModified": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})?', re.I ) -TIMESTAMP_PATTERN = re.compile( - rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}" +JSON_PUBLISHED = re.compile( + rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})?', re.I ) +TIMESTAMP_PATTERN = re.compile(rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE})({TIME_TZ_RE})") # English, French, German, Indonesian and Turkish dates cache MONTHS = [ @@ -444,6 +447,10 @@ def img_search( return None +# strftime directives carrying time of day or time zone +TIME_TZ_DIRECTIVES = ("%H", "%I", "%M", "%S", "%f", "%z", "%Z", "%p", "%X", "%c") + + def pattern_search( text: str, date_pattern: re.Pattern[str], @@ -451,12 +458,21 @@ def pattern_search( ) -> str | None: "Look for date expressions using a regular expression on a string of text." match = date_pattern.search(text) - if match and is_valid_date( + if not match or not is_valid_date( match[1], "%Y-%m-%d", earliest=options.min, latest=options.max ): - LOGGER.debug("regex found: %s %s", date_pattern, match[0]) - return convert_date(match[1], "%Y-%m-%d", options.format) - return None + return None + LOGGER.debug("regex found: %s %s", date_pattern, match[0]) + # carry time of day and time zone through when the output format needs them + # (regexes only capture the date part, the optional suffix holds time/tz) + if match.lastindex and match.lastindex >= 2 and match[2]: + if any(directive in options.format for directive in TIME_TZ_DIRECTIVES): + full = custom_parse( + match[1] + match[2], options.format, options.min, options.max + ) + if full is not None: + return full + return convert_date(match[1], "%Y-%m-%d", options.format) def json_search( diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 5785084c..ed6cef13 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -741,6 +741,49 @@ def test_exact_date(): ) +def test_free_text_timezone(): + """Time of day and time zone must be preserved when dates are extracted + from free text / JSON via regexes (issue #174).""" + tzformat = "%Y-%m-%d %H:%M:%S%z" + + # JSON-LD datePublished with time and time zone + jsonld = ( + '

text

" + ) + assert ( + find_date(jsonld, outputformat=tzformat, original_date=True) + == "2024-11-06 08:37:00+0530" + ) + # default date-only output stays unchanged + assert find_date(jsonld, original_date=True) == "2024-11-06" + + # JSON-LD dateModified with time and time zone + jsonld_mod = ( + '

text

" + ) + assert ( + find_date(jsonld_mod, outputformat=tzformat, original_date=False) + == "2024-11-06 08:37:00+0530" + ) + + # timestamp found in free text of the body + freetext = ( + "

Published on 2024-11-06T08:37:00+05:30 " + "by someone

" + ) + assert ( + find_date(freetext, outputformat=tzformat, original_date=True) + == "2024-11-06 08:37:00+0530" + ) + assert find_date(freetext, original_date=True) == "2024-11-06" + + def test_is_valid_date(): """test internal date validation""" assert (