From d6b7cd31192dda547f1b3414e1b3b1094aab7ce9 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Sun, 28 Jun 2026 19:05:09 +0200 Subject: [PATCH 1/5] maintenance: check consistency and simplify code --- htmldate/core.py | 54 +++++++++++++++++--------------- htmldate/meta.py | 6 ++-- htmldate/utils.py | 8 ++--- htmldate/validators.py | 70 ++++++++++++++++++++++++++---------------- tests/unit_tests.py | 10 ++++-- 5 files changed, 85 insertions(+), 63 deletions(-) diff --git a/htmldate/core.py b/htmldate/core.py index e7baf346..c336f0c1 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -7,10 +7,10 @@ import re from collections import Counter -from collections.abc import Callable +from collections.abc import Callable, Iterable from copy import deepcopy from datetime import datetime -from functools import lru_cache, partial +from functools import partial from lxml.html import HtmlElement, tostring @@ -205,11 +205,12 @@ def examine_text( options: Extractor, ) -> str | None: "Prepare text and try to extract a date." - text = trim_text(text) + if len(text.strip()) <= MIN_SEGMENT_LEN: + return None + text = trim_text(text) if len(text) <= MIN_SEGMENT_LEN: return None - text = NON_DIGITS_REGEX.sub("", text[:MAX_SEGMENT_LEN]) return try_date_expr( text, options.format, options.extensive, options.min, options.max @@ -218,20 +219,28 @@ def examine_text( def examine_date_elements( tree: HtmlElement, - expression: str, + expression: str | Iterable[str], options: Extractor, ) -> str | None: - """Check HTML elements one by one for date expressions""" - elements = tree.xpath(expression) - if not elements or len(elements) > MAX_POSSIBLE_CANDIDATES: - return None + """Check HTML elements one by one for date expressions. - for elem in elements: - # try element text and link title (Blogspot) - for text in [elem.text_content(), elem.get("title", "")]: - attempt = examine_text(text, options) - if attempt: - return attempt + ``expression`` can be a single XPath or an iterable of them; in the latter + case they are tried in order and the first one to yield a match wins, so a + high-priority expression can be given alongside cheap fallbacks without an + extra tree traversal.""" + expressions = [expression] if isinstance(expression, str) else expression + + for expr in expressions: + elements = tree.xpath(expr) + if not elements or len(elements) > MAX_POSSIBLE_CANDIDATES: + continue + + for elem in elements: + # try element text and link title (Blogspot) + for text in [elem.text_content(), elem.get("title", "")]: + attempt = examine_text(text, options) + if attempt: + return attempt return None @@ -425,7 +434,6 @@ def search_pattern( return select_candidate(candidates, catch, yearpat, options) -@lru_cache(maxsize=CACHE_SIZE) def compare_reference( reference: int, expression: str, @@ -605,7 +613,8 @@ def search_normalized( ) bestmatch = select_candidate(normalized, YMD_PATTERN, YMD_YEAR, options) return filter_ymd_candidate( - bestmatch, pattern, copyear, options.format, options.min, options.max + bestmatch.groups() if bestmatch else None, + pattern, copyear, options.format, options.min, options.max ) @@ -655,7 +664,7 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) result = filter_ymd_candidate( - bestmatch, + bestmatch.groups() if bestmatch else None, patterns[0], copyear, options.format, @@ -686,7 +695,7 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) result = filter_ymd_candidate( - bestmatch, + bestmatch.groups() if bestmatch else None, DATESTRINGS_PATTERN, copyear, options.format, @@ -911,12 +920,7 @@ def find_date( result = ( examine_date_elements( search_tree, - date_expr, - options, - ) - or examine_date_elements( - search_tree, - ".//title|.//h1", + [date_expr, ".//title|.//h1"], options, ) or examine_time_elements(search_tree, options) diff --git a/htmldate/meta.py b/htmldate/meta.py index 87a34a99..aefe5d8f 100644 --- a/htmldate/meta.py +++ b/htmldate/meta.py @@ -4,9 +4,8 @@ import logging -from .core import compare_reference from .extractors import try_date_expr -from .validators import filter_ymd_candidate, is_valid_date, is_valid_format +from .validators import _parse_and_validate, filter_ymd_candidate, is_valid_format LOGGER = logging.getLogger(__name__) @@ -24,9 +23,8 @@ def reset_caches() -> None: """Reset all known LRU caches used to speed-up processing. This may release some memory.""" # htmldate - compare_reference.cache_clear() + _parse_and_validate.cache_clear() filter_ymd_candidate.cache_clear() - is_valid_date.cache_clear() is_valid_format.cache_clear() try_date_expr.cache_clear() # charset_normalizer diff --git a/htmldate/utils.py b/htmldate/utils.py index 538a0cec..416e719a 100644 --- a/htmldate/utils.py +++ b/htmldate/utils.py @@ -127,14 +127,12 @@ def fetch_url(url: str) -> str | None: url: URL of the page to fetch. Returns: - HTML code as string, or Urllib3 response object (headers + body), or empty string in case - the result is invalid, or None if there was a problem with the network. + HTML code as a string, or None if the network request failed, the + response status was not 200, or the payload was invalid. """ # send try: - # read by streaming chunks (stream=True, iter_content=xx) - # so we can stop downloading as soon as MAX_FILE_SIZE is reached response = HTTP_POOL.request("GET", url, timeout=30) except Exception as err: LOGGER.error("download error: %s %s", url, err) # sys.exc_info()[0] @@ -150,7 +148,7 @@ def fetch_url(url: str) -> str | None: def is_dubious_html(beginning: str) -> bool: - "Assess if the object is proper HTML (awith a corresponding tag or declaration)." + "Assess if the object is proper HTML (with a corresponding tag or declaration)." return "html" not in beginning diff --git a/htmldate/validators.py b/htmldate/validators.py index 2425f287..cbf6d227 100644 --- a/htmldate/validators.py +++ b/htmldate/validators.py @@ -17,39 +17,56 @@ LOGGER.debug("minimum date setting: %s", MIN_DATE) -@lru_cache(maxsize=CACHE_SIZE) +def _is_in_range(dateobject: datetime, earliest: datetime, latest: datetime) -> bool: + """Check whether a datetime falls within the configured time window.""" + if ( + earliest.year <= dateobject.year <= latest.year + and earliest.timestamp() <= dateobject.timestamp() <= latest.timestamp() + ): + return True + return False + + def is_valid_date( date_input: datetime | str | None, outputformat: str, earliest: datetime, latest: datetime, ) -> bool: - """Validate a string w.r.t. the chosen outputformat and basic heuristics""" - # safety check + """Validate a date w.r.t. the chosen outputformat and time boundaries.""" if date_input is None: return False - # try if date can be parsed using chosen outputformat + # datetime path: no format needed for parsing, no cache required if isinstance(date_input, datetime): - dateobject = date_input - else: - # speed-up - try: - if outputformat == "%Y-%m-%d": - dateobject = datetime( - int(date_input[:4]), int(date_input[5:7]), int(date_input[8:10]) - ) - # default - else: - dateobject = datetime.strptime(date_input, outputformat) - except ValueError: - return False + result = _is_in_range(date_input, earliest, latest) + if not result: + LOGGER.debug("date not valid: %s", date_input) + return result - # year first, then full validation: not newer than today or stored variable - if ( - earliest.year <= dateobject.year <= latest.year - and earliest.timestamp() <= dateobject.timestamp() <= latest.timestamp() - ): + # string path: parse then validate (cached on the string + format combo) + return _parse_and_validate(date_input, outputformat, earliest, latest) + + +@lru_cache(maxsize=CACHE_SIZE) +def _parse_and_validate( + date_input: str, + outputformat: str, + earliest: datetime, + latest: datetime, +) -> bool: + """Parse a date string and validate it against time boundaries.""" + try: + if outputformat == "%Y-%m-%d": + dateobject = datetime( + int(date_input[:4]), int(date_input[5:7]), int(date_input[8:10]) + ) + else: + dateobject = datetime.strptime(date_input, outputformat) + except ValueError: + return False + + if _is_in_range(dateobject, earliest, latest): return True LOGGER.debug("date not valid: %s", date_input) return False @@ -146,7 +163,7 @@ def compare_values(reference: int, attempt: str, options: Extractor) -> int: @lru_cache(maxsize=CACHE_SIZE) def filter_ymd_candidate( - bestmatch: re.Match[str], + bestmatch: tuple[str, ...] | None, pattern: re.Pattern[str], copyear: int, outputformat: str, @@ -155,9 +172,9 @@ def filter_ymd_candidate( ) -> str | None: """Filter free text candidates in the YMD format""" if bestmatch is not None: - pagedate = "-".join([bestmatch[1], bestmatch[2], bestmatch[3]]) + pagedate = "-".join([bestmatch[0], bestmatch[1], bestmatch[2]]) if is_valid_date(pagedate, "%Y-%m-%d", earliest=min_date, latest=max_date) and ( - copyear == 0 or int(bestmatch[1]) >= copyear + copyear == 0 or int(bestmatch[0]) >= copyear ): LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate) return convert_date(pagedate, "%Y-%m-%d", outputformat) @@ -169,10 +186,9 @@ def convert_date(datestring: str, inputformat: str, outputformat: str) -> str: # speed-up (%Y-%m-%d) if inputformat == outputformat: return datestring - # date object (speedup) + # datetime object passed directly (callers may violate the str annotation) if isinstance(datestring, datetime): return datestring.strftime(outputformat) - # normal dateobject = datetime.strptime(datestring, inputformat) return dateobject.strftime(outputformat) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 5785084c..47b35f3f 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -24,6 +24,7 @@ from htmldate.core import ( compare_reference, examine_date_elements, + examine_text, find_date, search_page, search_pattern, @@ -181,6 +182,12 @@ def test_input(): assert get_max_date("2020-02-20T13:30:00") == datetime.datetime(2020, 2, 20, 13, 30) +def test_examine_text(): + """test early-exit guards in examine_text""" + assert examine_text(" ab ", OPTIONS) is None + assert examine_text("a b c", OPTIONS) is None + + def test_sanity(): """Test if function arguments are interpreted and processed correctly.""" # XPath looking for date elements @@ -834,8 +841,7 @@ def test_convert_date(): """test date conversion""" assert convert_date("2016-11-18", "%Y-%m-%d", "%d %B %Y") == "18 November 2016" assert convert_date("18 November 2016", "%d %B %Y", "%Y-%m-%d") == "2016-11-18" - dateobject = datetime.datetime.strptime("2016-11-18", "%Y-%m-%d") - assert convert_date(dateobject, "%d %B %Y", "%Y-%m-%d") == "2016-11-18" + assert convert_date(datetime.datetime(2016, 11, 18), "%Y-%m-%d %H:%M:%S", "%Y-%m-%d") == "2016-11-18" def test_try_date_expr(): From fcc26472a194ae5305a3f7088cafcbc36c004fbe Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Sun, 28 Jun 2026 19:10:53 +0200 Subject: [PATCH 2/5] fix CI --- htmldate/core.py | 20 ++++++++++---------- tests/unit_tests.py | 5 ++++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/htmldate/core.py b/htmldate/core.py index c336f0c1..11fceb8c 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -53,7 +53,6 @@ TWO_COMP_REGEX, ) from .settings import ( - CACHE_SIZE, CLEANING_LIST, MAX_POSSIBLE_CANDIDATES, MAX_SEGMENT_LEN, @@ -614,7 +613,11 @@ def search_normalized( bestmatch = select_candidate(normalized, YMD_PATTERN, YMD_YEAR, options) return filter_ymd_candidate( bestmatch.groups() if bestmatch else None, - pattern, copyear, options.format, options.min, options.max + pattern, + copyear, + options.format, + options.min, + options.max, ) @@ -917,14 +920,11 @@ def find_date( # then look for expressions # and try time elements - result = ( - examine_date_elements( - search_tree, - [date_expr, ".//title|.//h1"], - options, - ) - or examine_time_elements(search_tree, options) - ) + result = examine_date_elements( + search_tree, + [date_expr, ".//title|.//h1"], + options, + ) or examine_time_elements(search_tree, options) if result is not None: return result diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 47b35f3f..13e72633 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -841,7 +841,10 @@ def test_convert_date(): """test date conversion""" assert convert_date("2016-11-18", "%Y-%m-%d", "%d %B %Y") == "18 November 2016" assert convert_date("18 November 2016", "%d %B %Y", "%Y-%m-%d") == "2016-11-18" - assert convert_date(datetime.datetime(2016, 11, 18), "%Y-%m-%d %H:%M:%S", "%Y-%m-%d") == "2016-11-18" + assert ( + convert_date(datetime.datetime(2016, 11, 18), "%Y-%m-%d %H:%M:%S", "%Y-%m-%d") + == "2016-11-18" + ) def test_try_date_expr(): From 10fe3238d8bc6bb5ae98d87ec96baf7fc8e18076 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Sun, 28 Jun 2026 19:39:11 +0200 Subject: [PATCH 3/5] fix mypy --- htmldate/core.py | 6 +----- htmldate/validators.py | 18 +++++++----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/htmldate/core.py b/htmldate/core.py index 11fceb8c..c7b5573c 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -204,9 +204,6 @@ def examine_text( options: Extractor, ) -> str | None: "Prepare text and try to extract a date." - if len(text.strip()) <= MIN_SEGMENT_LEN: - return None - text = trim_text(text) if len(text) <= MIN_SEGMENT_LEN: return None @@ -472,8 +469,7 @@ def examine_abbr_elements( # class elif elem.get("class") in CLASS_ATTRS: # other attributes - if "title" in elem.attrib: - trytext = elem.get("title") + if trytext := elem.get("title"): LOGGER.debug("abbr published-title found: %s", trytext) # shortcut if options.original: diff --git a/htmldate/validators.py b/htmldate/validators.py index cbf6d227..2d2b7501 100644 --- a/htmldate/validators.py +++ b/htmldate/validators.py @@ -19,12 +19,10 @@ def _is_in_range(dateobject: datetime, earliest: datetime, latest: datetime) -> bool: """Check whether a datetime falls within the configured time window.""" - if ( + return ( earliest.year <= dateobject.year <= latest.year and earliest.timestamp() <= dateobject.timestamp() <= latest.timestamp() - ): - return True - return False + ) def is_valid_date( @@ -58,18 +56,16 @@ def _parse_and_validate( """Parse a date string and validate it against time boundaries.""" try: if outputformat == "%Y-%m-%d": - dateobject = datetime( - int(date_input[:4]), int(date_input[5:7]), int(date_input[8:10]) - ) + dateobject = datetime.fromisoformat(date_input) else: dateobject = datetime.strptime(date_input, outputformat) except ValueError: return False - if _is_in_range(dateobject, earliest, latest): - return True - LOGGER.debug("date not valid: %s", date_input) - return False + result = _is_in_range(dateobject, earliest, latest) + if not result: + LOGGER.debug("date not valid: %s", date_input) + return result def validate_and_convert( From 00d081e5b9333083418ce4b46f9867e9ee21908c Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Sun, 5 Jul 2026 21:08:57 +0200 Subject: [PATCH 4/5] simplify code and tests further --- htmldate/cli.py | 8 +- htmldate/core.py | 76 +++++------ htmldate/extractors.py | 21 +-- htmldate/utils.py | 11 +- htmldate/validators.py | 14 +- tests/evaluation.py | 28 +--- tests/realworld_tests.py | 22 +--- tests/unit_tests.py | 275 ++++++++++++++------------------------- 8 files changed, 161 insertions(+), 294 deletions(-) diff --git a/htmldate/cli.py b/htmldate/cli.py index ac58f6c3..07121963 100644 --- a/htmldate/cli.py +++ b/htmldate/cli.py @@ -38,7 +38,7 @@ def parse_args(args: list[str]) -> argparse.Namespace: """Define parser for command-line arguments""" argsparser = argparse.ArgumentParser() argsparser.add_argument( - "-f", "--fast", help="fast mode: disable extensive search", action="store_false" + "-f", "--fast", help="fast mode: disable extensive search", action="store_true" ) argsparser.add_argument( "-i", @@ -96,9 +96,9 @@ def process_args(args: argparse.Namespace) -> None: else: with open(args.inputfile, mode="r", encoding="utf-8") as inputfile: for line in inputfile: - htmltext = fetch_url(line.strip()) - result = cli_examine(htmltext, args) - sys.stdout.write(f"{line.strip()}\t{result or 'None'}\n") + url = line.strip() + result = cli_examine(fetch_url(url), args) + sys.stdout.write(f"{url}\t{result or 'None'}\n") def main() -> None: diff --git a/htmldate/core.py b/htmldate/core.py index c7b5573c..76aa0d9e 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -7,7 +7,7 @@ import re from collections import Counter -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Sized from copy import deepcopy from datetime import datetime from functools import partial @@ -46,7 +46,6 @@ YYYYMM_PATTERN, YYYYMM_CATCH, MMYYYY_PATTERN, - MMYYYY_YEAR, SIMPLE_PATTERN, THREE_COMP_REGEX_A, THREE_COMP_REGEX_B, @@ -213,22 +212,23 @@ def examine_text( ) +def has_plausible_candidates(candidates: Sized) -> bool: + "Check that the number of candidates is neither zero nor excessive." + return 0 < len(candidates) <= MAX_POSSIBLE_CANDIDATES + + def examine_date_elements( tree: HtmlElement, expression: str | Iterable[str], options: Extractor, ) -> str | None: - """Check HTML elements one by one for date expressions. - - ``expression`` can be a single XPath or an iterable of them; in the latter - case they are tried in order and the first one to yield a match wins, so a - high-priority expression can be given alongside cheap fallbacks without an - extra tree traversal.""" + """Check HTML elements one by one for date expressions. ``expression`` can + be a single XPath or an iterable of them, tried in order.""" expressions = [expression] if isinstance(expression, str) else expression for expr in expressions: elements = tree.xpath(expr) - if not elements or len(elements) > MAX_POSSIBLE_CANDIDATES: + if not has_plausible_candidates(elements): continue for elem in elements: @@ -268,11 +268,7 @@ def examine_header( # loop through all meta elements for elem in tree.iterfind(".//meta"): # safeguard - if ( - not elem.attrib - or "content" not in elem.attrib - and "datetime" not in elem.attrib - ): + if "content" not in elem.attrib and "datetime" not in elem.attrib: continue # name attribute, most frequent if "name" in elem.attrib: @@ -298,8 +294,8 @@ def examine_header( LOGGER.debug("examining meta property: %s", logstring(elem)) attempt = tryfunc(elem.get("content")) if attempt is not None: - if (attribute in DATE_ATTRIBUTES and options.original) or ( - attribute in PROPERTY_MODIFIED and not options.original + if attribute in ( + DATE_ATTRIBUTES if options.original else PROPERTY_MODIFIED ): headerdate = attempt # hurts precision @@ -314,8 +310,10 @@ def examine_header( attempt = tryfunc(elem.get("datetime") or elem.get("content")) # store value if attempt is not None: - if (attribute in ITEMPROP_ATTRS_ORIGINAL and options.original) or ( - attribute in ITEMPROP_ATTRS_MODIFIED and not options.original + if attribute in ( + ITEMPROP_ATTRS_ORIGINAL + if options.original + else ITEMPROP_ATTRS_MODIFIED ): headerdate = attempt # put on hold: hurts precision @@ -368,7 +366,7 @@ def select_candidate( options: Extractor, ) -> re.Match[str] | None: """Select a candidate among the most frequent matches""" - if not occurrences or len(occurrences) > MAX_POSSIBLE_CANDIDATES: + if not has_plausible_candidates(occurrences): return None if len(occurrences) == 1: @@ -395,13 +393,14 @@ def select_candidate( # safety net: plausibility if all(validation): - # same number of occurrences: always take top of the pile? - if counts[0] == counts[1]: - match = catch.search(patterns[0]) - # safety net: newer date but up to 50% less frequent - elif years[1] != years[0] and counts[1] / counts[0] > 0.5: + # newer date but up to 50% less frequent: take it, unless counts are tied + if ( + counts[0] != counts[1] + and years[1] != years[0] + and counts[1] / counts[0] > 0.5 + ): match = catch.search(patterns[1]) - # not newer or hopefully not significant + # same number of occurrences, or not newer / not significant: top of the pile else: match = catch.search(patterns[0]) elif any(validation): @@ -450,7 +449,7 @@ def examine_abbr_elements( ) -> str | None: """Scan the page for abbr elements and check if their content contains an eligible date""" elements = tree.findall(".//abbr") - if 0 < len(elements) < MAX_POSSIBLE_CANDIDATES: + if has_plausible_candidates(elements): reference = 0 for elem in elements: # data-utime (mostly Facebook) @@ -469,7 +468,8 @@ def examine_abbr_elements( # class elif elem.get("class") in CLASS_ATTRS: # other attributes - if trytext := elem.get("title"): + trytext = elem.get("title") + if trytext is not None: LOGGER.debug("abbr published-title found: %s", trytext) # shortcut if options.original: @@ -506,7 +506,7 @@ def examine_time_elements( ) -> str | None: """Scan the page for time elements and check if their content contains an eligible date""" elements = tree.findall(".//time") - if 0 < len(elements) < MAX_POSSIBLE_CANDIDATES: + if has_plausible_candidates(elements): # scan all the tags and look for the newest one reference = 0 for elem in elements: @@ -515,11 +515,7 @@ def examine_time_elements( # go for datetime if len(datetime_attr) > 6: # shortcut: time pubdate - if ( - "pubdate" in elem.attrib - and elem.get("pubdate") == "pubdate" - and options.original - ): + if elem.get("pubdate") == "pubdate" and options.original: shortcut_flag = True LOGGER.debug("shortcut for time pubdate found: %s", datetime_attr) # shortcuts: class attribute @@ -589,8 +585,6 @@ def search_normalized( normalizer: Callable[[str], str], copyear: int, options: Extractor, - *, - incomplete: bool = False, ) -> str | None: """Filter plausible years, normalize each candidate to the YMD format, then select the best match and validate it (shared candidate-selection pipeline).""" @@ -600,7 +594,6 @@ def search_normalized( yearpat=yearpat, earliest=options.min, latest=options.max, - incomplete=incomplete, ) # revert DD-MM-YYYY patterns before sorting normalized = Counter( @@ -712,7 +705,6 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: lambda item: normalize_match(THREE_COMP_REGEX_B.match(item)), copyear, options, - incomplete=True, ) if result is not None: return result @@ -746,11 +738,10 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: result = search_normalized( htmlstring, MMYYYY_PATTERN, - MMYYYY_YEAR, + SELECT_YMD_YEAR, normalize_two_comp, copyear, options, - incomplete=options.original, ) if result is not None: return result @@ -768,7 +759,7 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: # catchall: copyright mention if copyear != 0: LOGGER.debug("using copyright year as default") - dateobject = datetime(int(copyear), 1, 1) + dateobject = datetime(copyear, 1, 1) return dateobject.strftime(options.format) # last resort: 1 component @@ -909,10 +900,7 @@ def find_date( LOGGER.error("lxml cleaner error") # define expressions + text_content - if extensive_search: - date_expr = SLOW_PREPEND + DATE_EXPRESSIONS - else: - date_expr = FAST_PREPEND + DATE_EXPRESSIONS + date_expr = (SLOW_PREPEND if extensive_search else FAST_PREPEND) + DATE_EXPRESSIONS # then look for expressions # and try time elements diff --git a/htmldate/extractors.py b/htmldate/extractors.py index bd0c470c..8773b61a 100644 --- a/htmldate/extractors.py +++ b/htmldate/extractors.py @@ -19,7 +19,7 @@ # own from .settings import CACHE_SIZE, MAX_SEGMENT_LEN -from .utils import Extractor, trim_text +from .utils import Extractor, remove_if_attached, trim_text from .validators import convert_date, correct_year, is_valid_date, validate_and_convert LOGGER = logging.getLogger(__name__) @@ -207,14 +207,13 @@ YYYYMM_PATTERN = re.compile(r"\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\D") YYYYMM_CATCH = re.compile(rf"({YEAR_RE})[/.-](1[0-2]|0[1-9])") MMYYYY_PATTERN = re.compile(r"\D([01]?[0-9][/.-][12][0-9]{3})\D") -MMYYYY_YEAR = re.compile(rf"({YEAR_RE})\D?$") SIMPLE_PATTERN = re.compile(rf"(? HtmlElement: """Delete unwanted sections of an HTML document.""" for subtree in DISCARD_EXPRESSIONS(tree): - subtree.getparent().remove(subtree) + remove_if_attached(subtree) return tree @@ -259,18 +258,23 @@ def regex_parse(string: str) -> datetime | None: try: day, month, year = ( int(match.group(groups[0])), - int(TEXT_MONTHS[match.group(groups[1]).lower().strip(".")]), + TEXT_MONTHS[match.group(groups[1]).lower().strip(".")], int(match.group(groups[2])), ) year = correct_year(year) day, month = try_swap_values(day, month) dateobject = datetime(year, month, day) - except ValueError: + except (KeyError, ValueError): return None LOGGER.debug("multilingual text found: %s", dateobject) return dateobject +def _parse_yyyymmdd(digits: str) -> datetime: + "Build a datetime from a leading 8-digit YYYYMMDD run." + return datetime(int(digits[:4]), int(digits[4:6]), int(digits[6:8])) + + def custom_parse( string: str, outputformat: str, min_date: datetime, max_date: datetime ) -> str | None: @@ -283,9 +287,7 @@ def custom_parse( # a. '201709011234' not covered by dateparser, and regex too slow if string[4:8].isdigit(): try: - candidate = datetime( - int(string[:4]), int(string[4:6]), int(string[6:8]) - ) + candidate = _parse_yyyymmdd(string) except ValueError: LOGGER.debug("8-digit error: %s", string[:8]) # return None # b. much faster than extensive parsing @@ -309,8 +311,7 @@ def custom_parse( match = YMD_NO_SEP_PATTERN.search(string) if match: try: - year, month, day = int(match[1][:4]), int(match[1][4:6]), int(match[1][6:8]) - candidate = datetime(year, month, day) + candidate = _parse_yyyymmdd(match[1]) except ValueError: LOGGER.debug("YYYYMMDD value error: %s", match[0]) else: diff --git a/htmldate/utils.py b/htmldate/utils.py index 416e719a..0cf8fbbf 100644 --- a/htmldate/utils.py +++ b/htmldate/utils.py @@ -229,6 +229,13 @@ def load_html(htmlobject: bytes | str | HtmlElement) -> HtmlElement | None: return tree +def remove_if_attached(element: HtmlElement) -> None: + "Remove an element from its parent, if it still has one." + parent = element.getparent() + if parent is not None: + parent.remove(element) + + def clean_html(tree: HtmlElement, elemlist: list[str]) -> HtmlElement: "Delete selected elements." for element in tree.iter(elemlist): @@ -237,9 +244,7 @@ def clean_html(tree: HtmlElement, elemlist: list[str]) -> HtmlElement: try: element.drop_tree() except AttributeError: # pragma: no cover - parent = element.getparent() - if parent is not None: - parent.remove(element) + remove_if_attached(element) return tree diff --git a/htmldate/validators.py b/htmldate/validators.py index 2d2b7501..b8343619 100644 --- a/htmldate/validators.py +++ b/htmldate/validators.py @@ -56,7 +56,8 @@ def _parse_and_validate( """Parse a date string and validate it against time boundaries.""" try: if outputformat == "%Y-%m-%d": - dateobject = datetime.fromisoformat(date_input) + # read the leading YYYY-MM-DD and ignore the rest + dateobject = datetime.fromisoformat(date_input[:10]) else: dateobject = datetime.strptime(date_input, outputformat) except ValueError: @@ -117,7 +118,6 @@ def plausible_year_filter( yearpat: re.Pattern[str], earliest: datetime, latest: datetime, - incomplete: bool = False, ) -> Counter[str]: """Filter the date patterns to find plausible years only""" occurrences = Counter(pattern.findall(htmlstring)) # slow! @@ -130,11 +130,9 @@ def plausible_year_filter( del occurrences[item] continue - lastdigits = year_match[1] - if not incomplete: - potential_year = int(lastdigits) - else: - potential_year = correct_year(int(lastdigits)) + # correct_year() is a no-op for already-4-digit years, so this also + # covers yearpat patterns that only ever capture 4 digits + potential_year = correct_year(int(year_match[1])) if not min_year <= potential_year <= max_year: LOGGER.debug("no potential year: %s", item) @@ -168,7 +166,7 @@ def filter_ymd_candidate( ) -> str | None: """Filter free text candidates in the YMD format""" if bestmatch is not None: - pagedate = "-".join([bestmatch[0], bestmatch[1], bestmatch[2]]) + pagedate = "-".join(bestmatch[:3]) if is_valid_date(pagedate, "%Y-%m-%d", earliest=min_date, latest=max_date) and ( copyear == 0 or int(bestmatch[0]) >= copyear ): diff --git a/tests/evaluation.py b/tests/evaluation.py index 7282ab52..c80c802a 100644 --- a/tests/evaluation.py +++ b/tests/evaluation.py @@ -6,12 +6,8 @@ import os import re -try: - from cchardet import detect -except ImportError: - from charset_normalizer import detect - from htmldate import find_date +from htmldate.utils import decode_file from htmldate.validators import convert_date # Optional third-party libraries, only needed for the full benchmark @@ -43,31 +39,13 @@ def load_document(filename): """load mock page from samples""" - htmlstring = "" # look for the right directory for directory in ("test_set", "cache", "eval"): mypath = os.path.join(TEST_DIR, directory, filename) if os.path.isfile(mypath): break - # open and convert the file to str - try: - with open(mypath, "r", encoding="utf-8") as inputf: - htmlstring = inputf.read() - # encoding/windows fix for the tests - except UnicodeDecodeError: - # read as binary - with open(mypath, "rb") as inputf: - htmlbinary = inputf.read() - guessed_encoding = detect(htmlbinary)["encoding"] - if guessed_encoding: - try: - htmlstring = htmlbinary.decode(guessed_encoding) - except UnicodeDecodeError: - pass - if not htmlstring: - # print("Encoding error, using binary:", mypath) - htmlstring = htmlbinary - return htmlstring + with open(mypath, "rb") as inputf: + return decode_file(inputf.read()) def run_htmldate_extensive(htmlstring): diff --git a/tests/realworld_tests.py b/tests/realworld_tests.py index 0e0ee129..7f1217ab 100644 --- a/tests/realworld_tests.py +++ b/tests/realworld_tests.py @@ -100,8 +100,6 @@ "https://www.mozilla.org/en-US/security/advisories/mfsa2024-17/": "mozilla.org.mfsa2024-17.html", } -MEDIACLOUD_PAGES = {"pagename": "thing.html"} - def load_mock_page(url, dirname="cache", dictname=MOCK_PAGES): """load mock page from samples""" @@ -109,11 +107,6 @@ def load_mock_page(url, dirname="cache", dictname=MOCK_PAGES): return inputf.read() -def load_mediacloud_page(url): - """load mediacloud page from samples""" - return load_mock_page(url, dirname="test_set", dictname=MEDIACLOUD_PAGES) - - def test_no_date(): "These pages should not return any date" assert ( @@ -765,7 +758,7 @@ def test_dependencies(): def test_cli(): "Test the command-line interface" - # third test: Linux and MacOS only + # Linux and MacOS only if os.name != "nt": testargs = [] args = parse_args(testargs) @@ -779,16 +772,3 @@ def test_cli(): process_args(args) assert f.getvalue() == "2017-07-12\n" sys.stdin = sys.__stdin__ - - -if __name__ == "__main__": - # meta - test_readme_examples() - test_dependencies() - - # module-level - test_no_date() - test_exact_date() - test_approximate_date() - test_search_html() - test_cli() diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 13e72633..437d2cc4 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -1,4 +1,3 @@ -# pylint:disable-msg=W1401 """ Unit tests for the htmldate library. """ @@ -784,6 +783,23 @@ def test_is_valid_date(): is_valid_date("202-01", OUTPUTFORMAT, earliest=MIN_DATE, latest=LATEST_POSSIBLE) is False ) + # trailing content beyond the 10-char YYYY-MM-DD shape must be ignored + # (read as 2020-01-01), not parsed as a time/offset by fromisoformat() + assert ( + is_valid_date( + "2020-01-01-01-01", OUTPUTFORMAT, earliest=MIN_DATE, latest=LATEST_POSSIBLE + ) + is True + ) + assert ( + is_valid_date( + "2020-01-01 12:00:00-01-01", + OUTPUTFORMAT, + earliest=MIN_DATE, + latest=LATEST_POSSIBLE, + ) + is True + ) assert ( is_valid_date("1922", "%Y", earliest=MIN_DATE, latest=LATEST_POSSIBLE) is False ) @@ -851,7 +867,6 @@ def test_try_date_expr(): """test date extraction via external package""" assert try_date_expr(None, OUTPUTFORMAT, False, MIN_DATE, LATEST_POSSIBLE) is None - find_date.extensive_search = False assert ( try_date_expr( "Fri, Sept 1, 2017", OUTPUTFORMAT, False, MIN_DATE, LATEST_POSSIBLE @@ -859,7 +874,6 @@ def test_try_date_expr(): is None ) - find_date.extensive_search = True assert ( try_date_expr( "Friday, September 01, 2017", OUTPUTFORMAT, True, MIN_DATE, LATEST_POSSIBLE @@ -951,10 +965,6 @@ def test_try_date_expr(): ) -# def test_header(): -# assert examine_header(tree, options) - - def test_compare_reference(): """test comparison function""" options = Extractor(False, LATEST_POSSIBLE, MIN_DATE, False, OUTPUTFORMAT) @@ -1050,12 +1060,7 @@ def test_regex_parse(): custom_parse("3/14/2016", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None ) assert custom_parse("20041212", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None - assert custom_parse("20041212", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None assert custom_parse("1212-20-04", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is None - assert custom_parse("1212-20-04", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is None - assert ( - custom_parse("2004-12-12", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None - ) assert ( custom_parse("2004-12-12", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None ) @@ -1064,7 +1069,6 @@ def test_regex_parse(): custom_parse("12.12.2004", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None ) assert custom_parse("2019 28 meh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is None - assert custom_parse("2019 28 meh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is None # regex-based matches assert ( custom_parse("abcd 20041212 efgh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) @@ -1078,10 +1082,6 @@ def test_regex_parse(): custom_parse("abcd 2004-2 efgh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) is not None ) - assert ( - custom_parse("abcd 2004-2 efgh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE) - is not None - ) assert ( custom_parse( "abcd 32. Januar 2020 efgh", OUTPUTFORMAT, MIN_DATE, LATEST_POSSIBLE @@ -1110,129 +1110,84 @@ def test_regex_parse(): is None ) # for Nones caused by newlines and duplicates - assert regex_parse("January 1st, 1998") is not None - assert regex_parse("February 1st, 1998") is not None - assert regex_parse("March 1st, 1998") is not None - assert regex_parse("April 1st, 1998") is not None - assert regex_parse("May 1st, 1998") is not None - assert regex_parse("June 1st, 1998") is not None - assert regex_parse("July 1st, 1998") is not None - assert regex_parse("August 1st, 1998") is not None - assert regex_parse("September 1st, 1998") is not None - assert regex_parse("October 1st, 1998") is not None - assert regex_parse("November 1st, 1998") is not None - assert regex_parse("December 1st, 1998") is not None - assert regex_parse("Jan 1st, 1998") is not None - assert regex_parse("Feb 1st, 1998") is not None - assert regex_parse("Mar 1st, 1998") is not None - assert regex_parse("Apr 1st, 1998") is not None - assert regex_parse("Jun 1st, 1998") is not None - assert regex_parse("Jul 1st, 1998") is not None - assert regex_parse("Aug 1st, 1998") is not None - assert regex_parse("Sep 1st, 1998") is not None - assert regex_parse("Oct 1st, 1998") is not None - assert regex_parse("Nov 1st, 1998") is not None - assert regex_parse("Dec 1st, 1998") is not None - assert regex_parse("Januar 1, 1998") is not None - assert regex_parse("Jänner 1, 1998") is not None - assert regex_parse("Februar 1, 1998") is not None - assert regex_parse("Feber 1, 1998") is not None - assert regex_parse("März 1, 1998") is not None - assert regex_parse("April 1, 1998") is not None - assert regex_parse("Mai 1, 1998") is not None - assert regex_parse("Juni 1, 1998") is not None - assert regex_parse("Juli 1, 1998") is not None - assert regex_parse("August 1, 1998") is not None - assert regex_parse("September 1, 1998") is not None - assert regex_parse("Oktober 1, 1998") is not None + en_full = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ] + en_abbr = [ + "Jan", + "Feb", + "Mar", + "Apr", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ] + de_full = [ + "Januar", + "Jänner", + "Februar", + "Feber", + "März", + "April", + "Mai", + "Juni", + "Juli", + "August", + "September", + "Oktober", + "November", + "Dezember", + ] + tr_full = [ + "Ocak", + "Şubat", + "Mart", + "Nisan", + "Mayıs", + "Haziran", + "Temmuz", + "Ağustos", + "Eylül", + "Ekim", + "Kasım", + "Aralık", + ] + tr_abbr = [ + "Oca", + "Şub", + "Mar", + "Nis", + "May", + "Haz", + "Tem", + "Ağu", + "Eyl", + "Eki", + "Kas", + "Ara", + ] + for month in en_full + en_abbr: + assert regex_parse(f"{month} 1st, 1998") is not None, month + for month in de_full + tr_full + tr_abbr: + assert regex_parse(f"{month} 1, 1998") is not None, month assert regex_parse("1. Okt. 1998") is not None - assert regex_parse("November 1, 1998") is not None - assert regex_parse("Dezember 1, 1998") is not None - assert regex_parse("Ocak 1, 1998") is not None - assert regex_parse("Şubat 1, 1998") is not None - assert regex_parse("Mart 1, 1998") is not None - assert regex_parse("Nisan 1, 1998") is not None - assert regex_parse("Mayıs 1, 1998") is not None - assert regex_parse("Haziran 1, 1998") is not None - assert regex_parse("Temmuz 1, 1998") is not None - assert regex_parse("Ağustos 1, 1998") is not None - assert regex_parse("Eylül 1, 1998") is not None - assert regex_parse("Ekim 1, 1998") is not None - assert regex_parse("Kasım 1, 1998") is not None - assert regex_parse("Aralık 1, 1998") is not None - assert regex_parse("Oca 1, 1998") is not None - assert regex_parse("Şub 1, 1998") is not None - assert regex_parse("Mar 1, 1998") is not None - assert regex_parse("Nis 1, 1998") is not None - assert regex_parse("May 1, 1998") is not None - assert regex_parse("Haz 1, 1998") is not None - assert regex_parse("Tem 1, 1998") is not None - assert regex_parse("Ağu 1, 1998") is not None - assert regex_parse("Eyl 1, 1998") is not None - assert regex_parse("Eki 1, 1998") is not None - assert regex_parse("Kas 1, 1998") is not None - assert regex_parse("Ara 1, 1998") is not None - assert regex_parse("1 January 1998") is not None - assert regex_parse("1 February 1998") is not None - assert regex_parse("1 March 1998") is not None - assert regex_parse("1 April 1998") is not None - assert regex_parse("1 May 1998") is not None - assert regex_parse("1 June 1998") is not None - assert regex_parse("1 July 1998") is not None - assert regex_parse("1 August 1998") is not None - assert regex_parse("1 September 1998") is not None - assert regex_parse("1 October 1998") is not None - assert regex_parse("1 November 1998") is not None - assert regex_parse("1 December 1998") is not None - assert regex_parse("1 Jan 1998") is not None - assert regex_parse("1 Feb 1998") is not None - assert regex_parse("1 Mar 1998") is not None - assert regex_parse("1 Apr 1998") is not None - assert regex_parse("1 Jun 1998") is not None - assert regex_parse("1 Jul 1998") is not None - assert regex_parse("1 Aug 1998") is not None - assert regex_parse("1 Sep 1998") is not None - assert regex_parse("1 Oct 1998") is not None - assert regex_parse("1 Nov 1998") is not None - assert regex_parse("1 Dec 1998") is not None - assert regex_parse("1 Januar 1998") is not None - assert regex_parse("1 Jänner 1998") is not None - assert regex_parse("1 Februar 1998") is not None - assert regex_parse("1 Feber 1998") is not None - assert regex_parse("1 März 1998") is not None - assert regex_parse("1 April 1998") is not None - assert regex_parse("1 Mai 1998") is not None - assert regex_parse("1 Juni 1998") is not None - assert regex_parse("1 Juli 1998") is not None - assert regex_parse("1 August 1998") is not None - assert regex_parse("1 September 1998") is not None - assert regex_parse("1 Oktober 1998") is not None - assert regex_parse("1 November 1998") is not None - assert regex_parse("1 Dezember 1998") is not None - assert regex_parse("1 Ocak 1998") is not None - assert regex_parse("1 Şubat 1998") is not None - assert regex_parse("1 Mart 1998") is not None - assert regex_parse("1 Nisan 1998") is not None - assert regex_parse("1 Mayıs 1998") is not None - assert regex_parse("1 Haziran 1998") is not None - assert regex_parse("1 Temmuz 1998") is not None - assert regex_parse("1 Ağustos 1998") is not None - assert regex_parse("1 Eylül 1998") is not None - assert regex_parse("1 Ekim 1998") is not None - assert regex_parse("1 Kasım 1998") is not None - assert regex_parse("1 Aralık 1998") is not None - assert regex_parse("1 Oca 1998") is not None - assert regex_parse("1 Şub 1998") is not None - assert regex_parse("1 Mar 1998") is not None - assert regex_parse("1 Nis 1998") is not None - assert regex_parse("1 May 1998") is not None - assert regex_parse("1 Haz 1998") is not None - assert regex_parse("1 Tem 1998") is not None - assert regex_parse("1 Ağu 1998") is not None - assert regex_parse("1 Eyl 1998") is not None - assert regex_parse("1 Eki 1998") is not None - assert regex_parse("1 Kas 1998") is not None - assert regex_parse("1 Ara 1998") is not None + for month in en_full + en_abbr + de_full + tr_full + tr_abbr: + assert regex_parse(f"1 {month} 1998") is not None, month def test_external_date_parser(): @@ -1663,14 +1618,14 @@ def test_parser(): "https://www.example.org", ] args = parse_args(testargs) - assert args.fast is True + assert args.fast is False assert args.original is True assert args.verbose is True assert args.maxdate == "2015-12-31" assert args.URL == "https://www.example.org" testargs = ["-min", "2015-12-31"] args = parse_args(testargs) - assert args.fast is True + assert args.fast is False assert args.original is False assert args.verbose is False assert args.mindate == "2015-12-31" @@ -1851,41 +1806,3 @@ def test_deferred(): """ assert find_date(htmlstring, deferred_url_extractor=True) == "2017-09-01" assert find_date(htmlstring, deferred_url_extractor=False) == "2017-08-30" - - -if __name__ == "__main__": - # function-level - test_input() - test_sanity() - test_is_valid_date() - test_search_pattern() - test_try_date_expr() - test_convert_date() - test_compare_reference() - test_candidate_selection() - test_regex_parse() - test_external_date_parser() - # test_header() - - # module-level - test_deferred() - test_no_date() - test_exact_date() - test_search_html() - test_copyright_redos() - test_url() - test_approximate_url() - test_idiosyncrasies() - # new_pages() - - # dependencies - test_dependencies() - - # cli - test_parser() - test_cli() - - # loading functions - test_download() - test_encoding_detection() - test_fetch_url_errors() From 41a1bf8852a79b0ac344ec2416026803742279bc Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Wed, 8 Jul 2026 17:20:15 +0200 Subject: [PATCH 5/5] simplify further --- htmldate/core.py | 278 ++++++++++++++++++----------------------- htmldate/extractors.py | 103 ++++++--------- htmldate/validators.py | 16 +-- tests/unit_tests.py | 14 ++- 4 files changed, 175 insertions(+), 236 deletions(-) diff --git a/htmldate/core.py b/htmldate/core.py index 76aa0d9e..ed3a134b 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -23,33 +23,15 @@ json_search, regex_parse, pattern_search, - try_date_expr, + try_date_expr_opts, DATE_EXPRESSIONS, FAST_PREPEND, SLOW_PREPEND, FREE_TEXT_EXPRESSIONS, - YEAR_PATTERN, YMD_PATTERN, - COPYRIGHT_PATTERN, - TIMESTAMP_PATTERN, - THREE_PATTERN, - THREE_CATCH, - THREE_LOOSE_PATTERN, - THREE_LOOSE_CATCH, - SELECT_YMD_PATTERN, - SELECT_YMD_YEAR, - YMD_YEAR, - DATESTRINGS_PATTERN, - DATESTRINGS_CATCH, - SLASHES_PATTERN, - SLASHES_YEAR, - YYYYMM_PATTERN, - YYYYMM_CATCH, - MMYYYY_PATTERN, - SIMPLE_PATTERN, - THREE_COMP_REGEX_A, - THREE_COMP_REGEX_B, - TWO_COMP_REGEX, + DAY_RE, + MONTH_RE, + YEAR_RE, ) from .settings import ( CLEANING_LIST, @@ -68,6 +50,7 @@ is_valid_date, is_valid_format, plausible_year_filter, + update_reference, validate_and_convert, ) @@ -192,6 +175,44 @@ def logstring(element: HtmlElement) -> str: NON_DIGITS_REGEX = re.compile(r"\D+$") +# search_page / find_date patterns +TIMESTAMP_PATTERN = re.compile( + rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}" +) + +# component patterns +THREE_COMP_REGEX_A = re.compile(rf"({DAY_RE})[/.-]({MONTH_RE})[/.-]({YEAR_RE})") +THREE_COMP_REGEX_B = re.compile( + rf"({DAY_RE})/({MONTH_RE})/([0-9]{{2}})|({DAY_RE})[.-]({MONTH_RE})[.-]([0-9]{{2}})" +) +TWO_COMP_REGEX = re.compile(rf"({MONTH_RE})[/.-]({YEAR_RE})") + +# extensive search patterns +YEAR_PATTERN = re.compile(rf"^\D?({YEAR_RE})") +# bounded gap (\D{0,99}, not unbounded \D*) to avoid quadratic backtracking (ReDoS) +COPYRIGHT_PATTERN = re.compile( + rf"(?:©|\©|Copyright|\(c\))\D{{0,99}}(?:{YEAR_RE})?-?({YEAR_RE})\D" +) +THREE_PATTERN = re.compile(r"/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]") +THREE_CATCH = re.compile(r"([0-9]{4})/([0-9]{2})/([0-9]{2})") +THREE_LOOSE_PATTERN = re.compile(r"\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\D") +THREE_LOOSE_CATCH = re.compile(r"([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})") +SELECT_YMD_PATTERN = re.compile(r"\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\D") +SELECT_YMD_YEAR = re.compile(rf"({YEAR_RE})\D?$") +YMD_YEAR = re.compile(rf"^({YEAR_RE})") +DATESTRINGS_PATTERN = re.compile( + r"(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)" +) +DATESTRINGS_CATCH = re.compile(rf"({YEAR_RE})([01][0-9])([0-3][0-9])") +SLASHES_PATTERN = re.compile( + r"\D([0-3]?[0-9]/[01]?[0-9]/[0129][0-9]|[0-3][0-9]\.[01][0-9]\.[0129][0-9])\D" +) +SLASHES_YEAR = re.compile(r"([0-9]{2})$") +YYYYMM_PATTERN = re.compile(r"\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\D") +YYYYMM_CATCH = re.compile(rf"({YEAR_RE})[/.-](1[0-2]|0[1-9])") +MMYYYY_PATTERN = re.compile(r"\D([01]?[0-9][/.-][12][0-9]{3})\D") +SIMPLE_PATTERN = re.compile(rf"(? bool: @@ -258,41 +277,36 @@ def examine_header( """ headerdate, reserve = None, None - tryfunc = partial( - try_date_expr, - outputformat=options.format, - extensive_search=options.extensive, - min_date=options.min, - max_date=options.max, - ) + tryfunc = partial(try_date_expr_opts, options=options) # loop through all meta elements for elem in tree.iterfind(".//meta"): # safeguard if "content" not in elem.attrib and "datetime" not in elem.attrib: continue + content = elem.get("content") # name attribute, most frequent if "name" in elem.attrib: attribute = elem.get("name", "").lower() # url if attribute == "og:url": - reserve = extract_url_date(elem.get("content"), options) + reserve = extract_url_date(content, options) # date elif attribute in DATE_ATTRIBUTES: LOGGER.debug("examining meta name: %s", logstring(elem)) - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) # modified elif attribute in NAME_MODIFIED: LOGGER.debug("examining meta name: %s", logstring(elem)) if not options.original: - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) else: - reserve = tryfunc(elem.get("content")) + reserve = tryfunc(content) # property attribute elif "property" in elem.attrib: attribute = elem.get("property", "").lower() if attribute in DATE_ATTRIBUTES or attribute in PROPERTY_MODIFIED: LOGGER.debug("examining meta property: %s", logstring(elem)) - attempt = tryfunc(elem.get("content")) + attempt = tryfunc(content) if attempt is not None: if attribute in ( DATE_ATTRIBUTES if options.original else PROPERTY_MODIFIED @@ -307,7 +321,7 @@ def examine_header( # original: store / updated: override date if attribute in ITEMPROP_ATTRS: LOGGER.debug("examining meta itemprop: %s", logstring(elem)) - attempt = tryfunc(elem.get("datetime") or elem.get("content")) + attempt = tryfunc(elem.get("datetime") or content) # store value if attempt is not None: if attribute in ( @@ -322,8 +336,8 @@ def examine_header( # reserve with copyrightyear elif attribute == "copyrightyear": LOGGER.debug("examining meta itemprop: %s", logstring(elem)) - if "content" in elem.attrib: - attempt = "-".join([elem.get("content", ""), "01", "01"]) + if content is not None: + attempt = "-".join([content, "01", "01"]) if is_valid_date( attempt, "%Y-%m-%d", earliest=options.min, latest=options.max ): @@ -332,22 +346,18 @@ def examine_header( elif "pubdate" in elem.attrib: if elem.get("pubdate", "").lower() == "pubdate": LOGGER.debug("examining meta pubdate: %s", logstring(elem)) - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) # http-equiv, rare elif "http-equiv" in elem.attrib: attribute = elem.get("http-equiv", "").lower() - if attribute == "date": + if attribute in ("date", "last-modified"): LOGGER.debug("examining meta http-equiv: %s", logstring(elem)) - if options.original: - headerdate = tryfunc(elem.get("content")) + attempt = tryfunc(content) + # "date" is original, "last-modified" is updated + if (attribute == "date") == options.original: + headerdate = attempt else: - reserve = tryfunc(elem.get("content")) - elif attribute == "last-modified": - LOGGER.debug("examining meta http-equiv: %s", logstring(elem)) - if not options.original: - headerdate = tryfunc(elem.get("content")) - else: - reserve = tryfunc(elem.get("content")) + reserve = attempt # exit loop if headerdate is not None: break @@ -435,9 +445,7 @@ def compare_reference( options: Extractor, ) -> int: """Compare candidate to current date reference (includes date validation and older/newer test)""" - attempt = try_date_expr( - expression, options.format, options.extensive, options.min, options.max - ) + attempt = try_date_expr_opts(expression, options) if attempt is not None: return compare_values(reference, attempt, options) return reference @@ -459,12 +467,7 @@ def examine_abbr_elements( except ValueError: continue LOGGER.debug("data-utime found: %s", candidate) - # look for original date - if options.original and (reference == 0 or candidate < reference): - reference = candidate - # look for newest (i.e. largest time delta) - elif not options.original and candidate > reference: - reference = candidate + reference = update_reference(reference, candidate, options.original) # class elif elem.get("class") in CLASS_ATTRS: # other attributes @@ -473,13 +476,7 @@ def examine_abbr_elements( LOGGER.debug("abbr published-title found: %s", trytext) # shortcut if options.original: - attempt = try_date_expr( - trytext, - options.format, - options.extensive, - options.min, - options.max, - ) + attempt = try_date_expr_opts(trytext, options) if attempt is not None: return attempt else: @@ -510,47 +507,27 @@ def examine_time_elements( # scan all the tags and look for the newest one reference = 0 for elem in elements: - shortcut_flag = False datetime_attr = elem.get("datetime", "") # go for datetime if len(datetime_attr) > 6: - # shortcut: time pubdate - if elem.get("pubdate") == "pubdate" and options.original: - shortcut_flag = True - LOGGER.debug("shortcut for time pubdate found: %s", datetime_attr) - # shortcuts: class attribute - elif "class" in elem.attrib: - class_attr = elem.get("class", "") - if options.original and ( - class_attr.startswith("entry-date") - or class_attr.startswith("entry-time") - ): - shortcut_flag = True - LOGGER.debug( - "shortcut for time/datetime found: %s", datetime_attr - ) - # updated time - elif not options.original and class_attr == "updated": - shortcut_flag = True - LOGGER.debug( - "shortcut for updated time/datetime found: %s", - datetime_attr, - ) - # datetime attribute + class_attr = elem.get("class", "") + # mode-specific shortcut attributes + if options.original: + shortcut_flag = elem.get( + "pubdate" + ) == "pubdate" or class_attr.startswith( + ("entry-date", "entry-time") + ) else: - LOGGER.debug("time/datetime found: %s", datetime_attr) + shortcut_flag = class_attr == "updated" # analyze attribute if shortcut_flag: - attempt = try_date_expr( - datetime_attr, - options.format, - options.extensive, - options.min, - options.max, - ) + LOGGER.debug("shortcut for time/datetime found: %s", datetime_attr) + attempt = try_date_expr_opts(datetime_attr, options) if attempt is not None: return attempt else: + LOGGER.debug("time/datetime found: %s", datetime_attr) reference = compare_reference(reference, datetime_attr, options) # bare text in element elif elem.text is not None and len(elem.text) > 6: @@ -600,9 +577,15 @@ def search_normalized( {normalizer(item): count for item, count in candidates.items()} ) bestmatch = select_candidate(normalized, YMD_PATTERN, YMD_YEAR, options) + return _filter_ymd(bestmatch, copyear, options) + + +def _filter_ymd( + bestmatch: re.Match[str] | None, copyear: int, options: Extractor +) -> str | None: + "Convert the match to YMD groups (kept outside the lru_cache) and validate." return filter_ymd_candidate( bestmatch.groups() if bestmatch else None, - pattern, copyear, options.format, options.min, @@ -610,6 +593,29 @@ def search_normalized( ) +def _search_and_filter( + htmlstring: str, + pattern: re.Pattern[str], + catch: re.Pattern[str], + copyear: int, + options: Extractor, +) -> str | None: + "Search for a date pattern, then validate the best match in the YMD format." + bestmatch = search_pattern(htmlstring, pattern, catch, YEAR_PATTERN, options) + return _filter_ymd(bestmatch, copyear, options) + + +def _finalize_candidate( + dateobject: datetime | None, copyear: int, options: Extractor +) -> str | None: + "Apply the copyright-year floor, then validate and convert a candidate date." + if dateobject is not None and (copyear == 0 or dateobject.year >= copyear): + return validate_and_convert( + dateobject, options.format, earliest=options.min, latest=options.max + ) + return None + + def search_page(htmlstring: str, options: Extractor) -> str | None: """ Opportunistically search the HTML text for common text patterns @@ -648,20 +654,8 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: # target URL characteristics # then more loosely structured data for patterns in THREE_COMP_PATTERNS: - bestmatch = search_pattern( - htmlstring, - patterns[0], - patterns[1], - YEAR_PATTERN, - options, - ) - result = filter_ymd_candidate( - bestmatch.groups() if bestmatch else None, - patterns[0], - copyear, - options.format, - options.min, - options.max, + result = _search_and_filter( + htmlstring, patterns[0], patterns[1], copyear, options ) if result is not None: return result @@ -679,20 +673,8 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: return result # valid dates strings - bestmatch = search_pattern( - htmlstring, - DATESTRINGS_PATTERN, - DATESTRINGS_CATCH, - YEAR_PATTERN, - options, - ) - result = filter_ymd_candidate( - bestmatch.groups() if bestmatch else None, - DATESTRINGS_PATTERN, - copyear, - options.format, - options.min, - options.max, + result = _search_and_filter( + htmlstring, DATESTRINGS_PATTERN, DATESTRINGS_CATCH, copyear, options ) if result is not None: return result @@ -720,19 +702,11 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) if bestmatch is not None: - dateobject = datetime(int(bestmatch[1]), int(bestmatch[2]), 1) - if copyear == 0 or dateobject.year >= copyear: - result = validate_and_convert( - dateobject, options.format, earliest=options.min, latest=options.max - ) - if result is not None: - LOGGER.debug( - 'date found for pattern "%s": %s, %s', - YYYYMM_PATTERN, - bestmatch[1], - bestmatch[2], - ) - return result + result = _finalize_candidate( + datetime(int(bestmatch[1]), int(bestmatch[2]), 1), copyear, options + ) + if result is not None: + return result # 2 components, second option result = search_normalized( @@ -747,14 +721,10 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: return result # try full-blown text regex on all HTML? - text_date = regex_parse(htmlstring) # todo: find all candidates and disambiguate? - if copyear == 0 or (text_date and text_date.year >= copyear): - result = validate_and_convert( - text_date, options.format, earliest=options.min, latest=options.max - ) - if result is not None: - return result + result = _finalize_candidate(regex_parse(htmlstring), copyear, options) + if result is not None: + return result # catchall: copyright mention if copyear != 0: @@ -772,17 +742,7 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) if bestmatch is not None: - dateobject = datetime(int(bestmatch[0]), 1, 1) - if ( - is_valid_date( - dateobject, "%Y-%m-%d", earliest=options.min, latest=options.max - ) - and dateobject.year >= copyear - ): - LOGGER.debug( - 'date found for pattern "%s": %s', SIMPLE_PATTERN, bestmatch[0] - ) - return dateobject.strftime(options.format) + return _finalize_candidate(datetime(int(bestmatch[0]), 1, 1), copyear, options) return None diff --git a/htmldate/extractors.py b/htmldate/extractors.py index 8773b61a..b8f7a9ef 100644 --- a/htmldate/extractors.py +++ b/htmldate/extractors.py @@ -126,9 +126,6 @@ JSON_PUBLISHED = re.compile( rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I ) -TIMESTAMP_PATTERN = re.compile( - rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}" -) # English, French, German, Indonesian and Turkish dates cache MONTHS = [ @@ -176,39 +173,6 @@ re.I, ) -# core patterns -THREE_COMP_REGEX_A = re.compile(rf"({DAY_RE})[/.-]({MONTH_RE})[/.-]({YEAR_RE})") -THREE_COMP_REGEX_B = re.compile( - rf"({DAY_RE})/({MONTH_RE})/([0-9]{{2}})|({DAY_RE})[.-]({MONTH_RE})[.-]([0-9]{{2}})" -) -TWO_COMP_REGEX = re.compile(rf"({MONTH_RE})[/.-]({YEAR_RE})") - -# extensive search patterns -YEAR_PATTERN = re.compile(rf"^\D?({YEAR_RE})") -# bounded gap (\D{0,99}, not unbounded \D*) to avoid quadratic backtracking (ReDoS) -COPYRIGHT_PATTERN = re.compile( - rf"(?:©|\©|Copyright|\(c\))\D{{0,99}}(?:{YEAR_RE})?-?({YEAR_RE})\D" -) -THREE_PATTERN = re.compile(r"/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]") -THREE_CATCH = re.compile(r"([0-9]{4})/([0-9]{2})/([0-9]{2})") -THREE_LOOSE_PATTERN = re.compile(r"\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\D") -THREE_LOOSE_CATCH = re.compile(r"([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})") -SELECT_YMD_PATTERN = re.compile(r"\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\D") -SELECT_YMD_YEAR = re.compile(rf"({YEAR_RE})\D?$") -YMD_YEAR = re.compile(rf"^({YEAR_RE})") -DATESTRINGS_PATTERN = re.compile( - r"(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)" -) -DATESTRINGS_CATCH = re.compile(rf"({YEAR_RE})([01][0-9])([0-3][0-9])") -SLASHES_PATTERN = re.compile( - r"\D([0-3]?[0-9]/[01]?[0-9]/[0129][0-9]|[0-3][0-9]\.[01][0-9]\.[0129][0-9])\D" -) -SLASHES_YEAR = re.compile(r"([0-9]{2})$") -YYYYMM_PATTERN = re.compile(r"\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\D") -YYYYMM_CATCH = re.compile(rf"({YEAR_RE})[/.-](1[0-2]|0[1-9])") -MMYYYY_PATTERN = re.compile(r"\D([01]?[0-9][/.-][12][0-9]{3})\D") -SIMPLE_PATTERN = re.compile(rf"(? HtmlElement: """Delete unwanted sections of an HTML document.""" @@ -241,6 +205,13 @@ def try_swap_values(day: int, month: int) -> tuple[int, int]: return (month, day) if month > 12 and day <= 12 else (day, month) +def _build_dmy(day: int, month: int, year: int) -> datetime: + "Build a datetime from day/month/year, fixing 2-digit years and day/month order." + year = correct_year(year) + day, month = try_swap_values(day, month) + return datetime(year, month, day) + + def regex_parse(string: str) -> datetime | None: """Try full-text parse for date elements using a series of regular expressions with particular emphasis on English, French, German and Turkish""" @@ -256,14 +227,11 @@ def regex_parse(string: str) -> datetime | None: ) # process and return try: - day, month, year = ( + dateobject = _build_dmy( int(match.group(groups[0])), - TEXT_MONTHS[match.group(groups[1]).lower().strip(".")], + TEXT_MONTHS[match.group(groups[1]).lower()], int(match.group(groups[2])), ) - year = correct_year(year) - day, month = try_swap_values(day, month) - dateobject = datetime(year, month, day) except (KeyError, ValueError): return None LOGGER.debug("multilingual text found: %s", dateobject) @@ -301,11 +269,11 @@ def custom_parse( except (OverflowError, TypeError, ValueError): LOGGER.debug("dateutil parsing error: %s", string) # c. plausibility test - if candidate is not None and ( - is_valid_date(candidate, outputformat, earliest=min_date, latest=max_date) - ): - LOGGER.debug("parsing result: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 2. Try YYYYMMDD, use regex match = YMD_NO_SEP_PATTERN.search(string) @@ -315,36 +283,36 @@ def custom_parse( except ValueError: LOGGER.debug("YYYYMMDD value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("YYYYMMDD match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 3. Try the very common YMD, Y-M-D, and D-M-Y patterns match = YMD_PATTERN.search(string) if match: try: if match.lastgroup == "day": - year, month, day = ( + candidate = datetime( int(match.group("year")), int(match.group("month")), int(match.group("day")), ) else: - day, month, year = ( + candidate = _build_dmy( int(match.group("day2")), int(match.group("month2")), int(match.group("year2")), ) - year = correct_year(year) - day, month = try_swap_values(day, month) - - candidate = datetime(year, month, day) except ValueError: # pragma: no cover LOGGER.debug("regex value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("regex match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 4. Try the Y-M and M-Y patterns match = YM_PATTERN.search(string) @@ -361,9 +329,11 @@ def custom_parse( except ValueError: LOGGER.debug("Y-M value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("Y-M match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 5. Try the other regex pattern dateobject = regex_parse(string) @@ -431,6 +401,13 @@ def try_date_expr( return None +def try_date_expr_opts(string: str | None, options: Extractor) -> str | None: + "Uncached options wrapper for try_date_expr (Extractor is unhashable)." + return try_date_expr( + string, options.format, options.extensive, options.min, options.max + ) + + def img_search( tree: HtmlElement, options: Extractor, @@ -492,9 +469,7 @@ def idiosyncrasies_search( if len(parts[0]) == 4: # year in first position candidate = datetime(int(parts[0]), int(parts[1]), int(parts[2])) else: # len(parts[2]) in (2, 4): # DD/MM/YY - day, month = try_swap_values(int(parts[0]), int(parts[1])) - year = correct_year(int(parts[2])) - candidate = datetime(year, month, day) + candidate = _build_dmy(int(parts[0]), int(parts[1]), int(parts[2])) return validate_and_convert( candidate, options.format, earliest=options.min, latest=options.max ) diff --git a/htmldate/validators.py b/htmldate/validators.py index b8343619..8fb96650 100644 --- a/htmldate/validators.py +++ b/htmldate/validators.py @@ -141,6 +141,13 @@ def plausible_year_filter( return occurrences +def update_reference(reference: int, candidate: int, original: bool) -> int: + "Fold a timestamp into the running reference: oldest if original, else newest." + if original: + return min(reference, candidate) if reference else candidate + return max(reference, candidate) + + def compare_values(reference: int, attempt: str, options: Extractor) -> int: """Compare the date expression to a reference""" try: @@ -148,17 +155,12 @@ def compare_values(reference: int, attempt: str, options: Extractor) -> int: except Exception as err: LOGGER.debug("datetime.strptime exception: %s for string %s", err, attempt) return reference - if options.original: - reference = min(reference, timestamp) if reference else timestamp - else: - reference = max(reference, timestamp) - return reference + return update_reference(reference, timestamp, options.original) @lru_cache(maxsize=CACHE_SIZE) def filter_ymd_candidate( bestmatch: tuple[str, ...] | None, - pattern: re.Pattern[str], copyear: int, outputformat: str, min_date: datetime, @@ -170,7 +172,7 @@ def filter_ymd_candidate( if is_valid_date(pagedate, "%Y-%m-%d", earliest=min_date, latest=max_date) and ( copyear == 0 or int(bestmatch[0]) >= copyear ): - LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate) + LOGGER.debug("date found: %s", pagedate) return convert_date(pagedate, "%Y-%m-%d", outputformat) return None diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 437d2cc4..14247585 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -290,12 +290,10 @@ def test_exact_date(): ) == "2017-01-09" ) - assert ( - find_date( - '' - ) - == "2017-01-01" - ) + # copyrightyear reserve, with and without extensive search + htmldoc = '' + for extensive in (True, False): + assert find_date(htmldoc, extensive_search=extensive) == "2017-01-01" # original date htmldoc = '' @@ -523,6 +521,10 @@ def test_exact_date(): ) == "2011-09-28" ) + # a shortcut