diff --git a/htmldate/cli.py b/htmldate/cli.py index ac58f6c3..07121963 100644 --- a/htmldate/cli.py +++ b/htmldate/cli.py @@ -38,7 +38,7 @@ def parse_args(args: list[str]) -> argparse.Namespace: """Define parser for command-line arguments""" argsparser = argparse.ArgumentParser() argsparser.add_argument( - "-f", "--fast", help="fast mode: disable extensive search", action="store_false" + "-f", "--fast", help="fast mode: disable extensive search", action="store_true" ) argsparser.add_argument( "-i", @@ -96,9 +96,9 @@ def process_args(args: argparse.Namespace) -> None: else: with open(args.inputfile, mode="r", encoding="utf-8") as inputfile: for line in inputfile: - htmltext = fetch_url(line.strip()) - result = cli_examine(htmltext, args) - sys.stdout.write(f"{line.strip()}\t{result or 'None'}\n") + url = line.strip() + result = cli_examine(fetch_url(url), args) + sys.stdout.write(f"{url}\t{result or 'None'}\n") def main() -> None: diff --git a/htmldate/core.py b/htmldate/core.py index e7baf346..ed3a134b 100644 --- a/htmldate/core.py +++ b/htmldate/core.py @@ -7,10 +7,10 @@ import re from collections import Counter -from collections.abc import Callable +from collections.abc import Callable, Iterable, Sized from copy import deepcopy from datetime import datetime -from functools import lru_cache, partial +from functools import partial from lxml.html import HtmlElement, tostring @@ -23,37 +23,17 @@ json_search, regex_parse, pattern_search, - try_date_expr, + try_date_expr_opts, DATE_EXPRESSIONS, FAST_PREPEND, SLOW_PREPEND, FREE_TEXT_EXPRESSIONS, - YEAR_PATTERN, YMD_PATTERN, - COPYRIGHT_PATTERN, - TIMESTAMP_PATTERN, - THREE_PATTERN, - THREE_CATCH, - THREE_LOOSE_PATTERN, - THREE_LOOSE_CATCH, - SELECT_YMD_PATTERN, - SELECT_YMD_YEAR, - YMD_YEAR, - DATESTRINGS_PATTERN, - DATESTRINGS_CATCH, - SLASHES_PATTERN, - SLASHES_YEAR, - YYYYMM_PATTERN, - YYYYMM_CATCH, - MMYYYY_PATTERN, - MMYYYY_YEAR, - SIMPLE_PATTERN, - THREE_COMP_REGEX_A, - THREE_COMP_REGEX_B, - TWO_COMP_REGEX, + DAY_RE, + MONTH_RE, + YEAR_RE, ) from .settings import ( - CACHE_SIZE, CLEANING_LIST, MAX_POSSIBLE_CANDIDATES, MAX_SEGMENT_LEN, @@ -70,6 +50,7 @@ is_valid_date, is_valid_format, plausible_year_filter, + update_reference, validate_and_convert, ) @@ -194,6 +175,44 @@ def logstring(element: HtmlElement) -> str: NON_DIGITS_REGEX = re.compile(r"\D+$") +# search_page / find_date patterns +TIMESTAMP_PATTERN = re.compile( + rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}" +) + +# component patterns +THREE_COMP_REGEX_A = re.compile(rf"({DAY_RE})[/.-]({MONTH_RE})[/.-]({YEAR_RE})") +THREE_COMP_REGEX_B = re.compile( + rf"({DAY_RE})/({MONTH_RE})/([0-9]{{2}})|({DAY_RE})[.-]({MONTH_RE})[.-]([0-9]{{2}})" +) +TWO_COMP_REGEX = re.compile(rf"({MONTH_RE})[/.-]({YEAR_RE})") + +# extensive search patterns +YEAR_PATTERN = re.compile(rf"^\D?({YEAR_RE})") +# bounded gap (\D{0,99}, not unbounded \D*) to avoid quadratic backtracking (ReDoS) +COPYRIGHT_PATTERN = re.compile( + rf"(?:©|\©|Copyright|\(c\))\D{{0,99}}(?:{YEAR_RE})?-?({YEAR_RE})\D" +) +THREE_PATTERN = re.compile(r"/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]") +THREE_CATCH = re.compile(r"([0-9]{4})/([0-9]{2})/([0-9]{2})") +THREE_LOOSE_PATTERN = re.compile(r"\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\D") +THREE_LOOSE_CATCH = re.compile(r"([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})") +SELECT_YMD_PATTERN = re.compile(r"\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\D") +SELECT_YMD_YEAR = re.compile(rf"({YEAR_RE})\D?$") +YMD_YEAR = re.compile(rf"^({YEAR_RE})") +DATESTRINGS_PATTERN = re.compile( + r"(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)" +) +DATESTRINGS_CATCH = re.compile(rf"({YEAR_RE})([01][0-9])([0-3][0-9])") +SLASHES_PATTERN = re.compile( + r"\D([0-3]?[0-9]/[01]?[0-9]/[0129][0-9]|[0-3][0-9]\.[01][0-9]\.[0129][0-9])\D" +) +SLASHES_YEAR = re.compile(r"([0-9]{2})$") +YYYYMM_PATTERN = re.compile(r"\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\D") +YYYYMM_CATCH = re.compile(rf"({YEAR_RE})[/.-](1[0-2]|0[1-9])") +MMYYYY_PATTERN = re.compile(r"\D([01]?[0-9][/.-][12][0-9]{3})\D") +SIMPLE_PATTERN = re.compile(rf"(? str | None: "Prepare text and try to extract a date." text = trim_text(text) - if len(text) <= MIN_SEGMENT_LEN: return None - text = NON_DIGITS_REGEX.sub("", text[:MAX_SEGMENT_LEN]) - return try_date_expr( - text, options.format, options.extensive, options.min, options.max - ) + return try_date_expr_opts(text, options) + + +def has_plausible_candidates(candidates: Sized) -> bool: + "Check that the number of candidates is neither zero nor excessive." + return 0 < len(candidates) <= MAX_POSSIBLE_CANDIDATES def examine_date_elements( tree: HtmlElement, - expression: str, + expression: str | Iterable[str], options: Extractor, ) -> str | None: - """Check HTML elements one by one for date expressions""" - elements = tree.xpath(expression) - if not elements or len(elements) > MAX_POSSIBLE_CANDIDATES: - return None + """Check HTML elements one by one for date expressions. ``expression`` can + be a single XPath or an iterable of them, tried in order.""" + expressions = [expression] if isinstance(expression, str) else expression - for elem in elements: - # try element text and link title (Blogspot) - for text in [elem.text_content(), elem.get("title", "")]: - attempt = examine_text(text, options) - if attempt: - return attempt + for expr in expressions: + elements = tree.xpath(expr) + if not has_plausible_candidates(elements): + continue + + for elem in elements: + # try element text and link title (Blogspot) + for text in [elem.text_content(), elem.get("title", "")]: + attempt = examine_text(text, options) + if attempt: + return attempt return None @@ -253,48 +277,39 @@ def examine_header( """ headerdate, reserve = None, None - tryfunc = partial( - try_date_expr, - outputformat=options.format, - extensive_search=options.extensive, - min_date=options.min, - max_date=options.max, - ) + tryfunc = partial(try_date_expr_opts, options=options) # loop through all meta elements for elem in tree.iterfind(".//meta"): # safeguard - if ( - not elem.attrib - or "content" not in elem.attrib - and "datetime" not in elem.attrib - ): + if "content" not in elem.attrib and "datetime" not in elem.attrib: continue + content = elem.get("content") # name attribute, most frequent if "name" in elem.attrib: attribute = elem.get("name", "").lower() # url if attribute == "og:url": - reserve = extract_url_date(elem.get("content"), options) + reserve = extract_url_date(content, options) # date elif attribute in DATE_ATTRIBUTES: LOGGER.debug("examining meta name: %s", logstring(elem)) - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) # modified elif attribute in NAME_MODIFIED: LOGGER.debug("examining meta name: %s", logstring(elem)) if not options.original: - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) else: - reserve = tryfunc(elem.get("content")) + reserve = tryfunc(content) # property attribute elif "property" in elem.attrib: attribute = elem.get("property", "").lower() if attribute in DATE_ATTRIBUTES or attribute in PROPERTY_MODIFIED: LOGGER.debug("examining meta property: %s", logstring(elem)) - attempt = tryfunc(elem.get("content")) + attempt = tryfunc(content) if attempt is not None: - if (attribute in DATE_ATTRIBUTES and options.original) or ( - attribute in PROPERTY_MODIFIED and not options.original + if attribute in ( + DATE_ATTRIBUTES if options.original else PROPERTY_MODIFIED ): headerdate = attempt # hurts precision @@ -306,11 +321,13 @@ def examine_header( # original: store / updated: override date if attribute in ITEMPROP_ATTRS: LOGGER.debug("examining meta itemprop: %s", logstring(elem)) - attempt = tryfunc(elem.get("datetime") or elem.get("content")) + attempt = tryfunc(elem.get("datetime") or content) # store value if attempt is not None: - if (attribute in ITEMPROP_ATTRS_ORIGINAL and options.original) or ( - attribute in ITEMPROP_ATTRS_MODIFIED and not options.original + if attribute in ( + ITEMPROP_ATTRS_ORIGINAL + if options.original + else ITEMPROP_ATTRS_MODIFIED ): headerdate = attempt # put on hold: hurts precision @@ -319,8 +336,8 @@ def examine_header( # reserve with copyrightyear elif attribute == "copyrightyear": LOGGER.debug("examining meta itemprop: %s", logstring(elem)) - if "content" in elem.attrib: - attempt = "-".join([elem.get("content", ""), "01", "01"]) + if content is not None: + attempt = "-".join([content, "01", "01"]) if is_valid_date( attempt, "%Y-%m-%d", earliest=options.min, latest=options.max ): @@ -329,22 +346,18 @@ def examine_header( elif "pubdate" in elem.attrib: if elem.get("pubdate", "").lower() == "pubdate": LOGGER.debug("examining meta pubdate: %s", logstring(elem)) - headerdate = tryfunc(elem.get("content")) + headerdate = tryfunc(content) # http-equiv, rare elif "http-equiv" in elem.attrib: attribute = elem.get("http-equiv", "").lower() - if attribute == "date": + if attribute in ("date", "last-modified"): LOGGER.debug("examining meta http-equiv: %s", logstring(elem)) - if options.original: - headerdate = tryfunc(elem.get("content")) + attempt = tryfunc(content) + # "date" is original, "last-modified" is updated + if (attribute == "date") == options.original: + headerdate = attempt else: - reserve = tryfunc(elem.get("content")) - elif attribute == "last-modified": - LOGGER.debug("examining meta http-equiv: %s", logstring(elem)) - if not options.original: - headerdate = tryfunc(elem.get("content")) - else: - reserve = tryfunc(elem.get("content")) + reserve = attempt # exit loop if headerdate is not None: break @@ -363,7 +376,7 @@ def select_candidate( options: Extractor, ) -> re.Match[str] | None: """Select a candidate among the most frequent matches""" - if not occurrences or len(occurrences) > MAX_POSSIBLE_CANDIDATES: + if not has_plausible_candidates(occurrences): return None if len(occurrences) == 1: @@ -390,13 +403,14 @@ def select_candidate( # safety net: plausibility if all(validation): - # same number of occurrences: always take top of the pile? - if counts[0] == counts[1]: - match = catch.search(patterns[0]) - # safety net: newer date but up to 50% less frequent - elif years[1] != years[0] and counts[1] / counts[0] > 0.5: + # newer date but up to 50% less frequent: take it, unless counts are tied + if ( + counts[0] != counts[1] + and years[1] != years[0] + and counts[1] / counts[0] > 0.5 + ): match = catch.search(patterns[1]) - # not newer or hopefully not significant + # same number of occurrences, or not newer / not significant: top of the pile else: match = catch.search(patterns[0]) elif any(validation): @@ -425,16 +439,13 @@ def search_pattern( return select_candidate(candidates, catch, yearpat, options) -@lru_cache(maxsize=CACHE_SIZE) def compare_reference( reference: int, expression: str, options: Extractor, ) -> int: """Compare candidate to current date reference (includes date validation and older/newer test)""" - attempt = try_date_expr( - expression, options.format, options.extensive, options.min, options.max - ) + attempt = try_date_expr_opts(expression, options) if attempt is not None: return compare_values(reference, attempt, options) return reference @@ -446,7 +457,7 @@ def examine_abbr_elements( ) -> str | None: """Scan the page for abbr elements and check if their content contains an eligible date""" elements = tree.findall(".//abbr") - if 0 < len(elements) < MAX_POSSIBLE_CANDIDATES: + if has_plausible_candidates(elements): reference = 0 for elem in elements: # data-utime (mostly Facebook) @@ -456,27 +467,16 @@ def examine_abbr_elements( except ValueError: continue LOGGER.debug("data-utime found: %s", candidate) - # look for original date - if options.original and (reference == 0 or candidate < reference): - reference = candidate - # look for newest (i.e. largest time delta) - elif not options.original and candidate > reference: - reference = candidate + reference = update_reference(reference, candidate, options.original) # class elif elem.get("class") in CLASS_ATTRS: # other attributes - if "title" in elem.attrib: - trytext = elem.get("title") + trytext = elem.get("title") + if trytext is not None: LOGGER.debug("abbr published-title found: %s", trytext) # shortcut if options.original: - attempt = try_date_expr( - trytext, - options.format, - options.extensive, - options.min, - options.max, - ) + attempt = try_date_expr_opts(trytext, options) if attempt is not None: return attempt else: @@ -503,55 +503,31 @@ def examine_time_elements( ) -> str | None: """Scan the page for time elements and check if their content contains an eligible date""" elements = tree.findall(".//time") - if 0 < len(elements) < MAX_POSSIBLE_CANDIDATES: + if has_plausible_candidates(elements): # scan all the tags and look for the newest one reference = 0 for elem in elements: - shortcut_flag = False datetime_attr = elem.get("datetime", "") # go for datetime if len(datetime_attr) > 6: - # shortcut: time pubdate - if ( - "pubdate" in elem.attrib - and elem.get("pubdate") == "pubdate" - and options.original - ): - shortcut_flag = True - LOGGER.debug("shortcut for time pubdate found: %s", datetime_attr) - # shortcuts: class attribute - elif "class" in elem.attrib: - class_attr = elem.get("class", "") - if options.original and ( - class_attr.startswith("entry-date") - or class_attr.startswith("entry-time") - ): - shortcut_flag = True - LOGGER.debug( - "shortcut for time/datetime found: %s", datetime_attr - ) - # updated time - elif not options.original and class_attr == "updated": - shortcut_flag = True - LOGGER.debug( - "shortcut for updated time/datetime found: %s", - datetime_attr, - ) - # datetime attribute + class_attr = elem.get("class", "") + # mode-specific shortcut attributes + if options.original: + shortcut_flag = elem.get( + "pubdate" + ) == "pubdate" or class_attr.startswith( + ("entry-date", "entry-time") + ) else: - LOGGER.debug("time/datetime found: %s", datetime_attr) + shortcut_flag = class_attr == "updated" # analyze attribute if shortcut_flag: - attempt = try_date_expr( - datetime_attr, - options.format, - options.extensive, - options.min, - options.max, - ) + LOGGER.debug("shortcut for time/datetime found: %s", datetime_attr) + attempt = try_date_expr_opts(datetime_attr, options) if attempt is not None: return attempt else: + LOGGER.debug("time/datetime found: %s", datetime_attr) reference = compare_reference(reference, datetime_attr, options) # bare text in element elif elem.text is not None and len(elem.text) > 6: @@ -586,8 +562,6 @@ def search_normalized( normalizer: Callable[[str], str], copyear: int, options: Extractor, - *, - incomplete: bool = False, ) -> str | None: """Filter plausible years, normalize each candidate to the YMD format, then select the best match and validate it (shared candidate-selection pipeline).""" @@ -597,18 +571,51 @@ def search_normalized( yearpat=yearpat, earliest=options.min, latest=options.max, - incomplete=incomplete, ) # revert DD-MM-YYYY patterns before sorting normalized = Counter( {normalizer(item): count for item, count in candidates.items()} ) bestmatch = select_candidate(normalized, YMD_PATTERN, YMD_YEAR, options) + return _filter_ymd(bestmatch, copyear, options) + + +def _filter_ymd( + bestmatch: re.Match[str] | None, copyear: int, options: Extractor +) -> str | None: + "Convert the match to YMD groups (kept outside the lru_cache) and validate." return filter_ymd_candidate( - bestmatch, pattern, copyear, options.format, options.min, options.max + bestmatch.groups() if bestmatch else None, + copyear, + options.format, + options.min, + options.max, ) +def _search_and_filter( + htmlstring: str, + pattern: re.Pattern[str], + catch: re.Pattern[str], + copyear: int, + options: Extractor, +) -> str | None: + "Search for a date pattern, then validate the best match in the YMD format." + bestmatch = search_pattern(htmlstring, pattern, catch, YEAR_PATTERN, options) + return _filter_ymd(bestmatch, copyear, options) + + +def _finalize_candidate( + dateobject: datetime | None, copyear: int, options: Extractor +) -> str | None: + "Apply the copyright-year floor, then validate and convert a candidate date." + if dateobject is not None and (copyear == 0 or dateobject.year >= copyear): + return validate_and_convert( + dateobject, options.format, earliest=options.min, latest=options.max + ) + return None + + def search_page(htmlstring: str, options: Extractor) -> str | None: """ Opportunistically search the HTML text for common text patterns @@ -647,20 +654,8 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: # target URL characteristics # then more loosely structured data for patterns in THREE_COMP_PATTERNS: - bestmatch = search_pattern( - htmlstring, - patterns[0], - patterns[1], - YEAR_PATTERN, - options, - ) - result = filter_ymd_candidate( - bestmatch, - patterns[0], - copyear, - options.format, - options.min, - options.max, + result = _search_and_filter( + htmlstring, patterns[0], patterns[1], copyear, options ) if result is not None: return result @@ -678,20 +673,8 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: return result # valid dates strings - bestmatch = search_pattern( - htmlstring, - DATESTRINGS_PATTERN, - DATESTRINGS_CATCH, - YEAR_PATTERN, - options, - ) - result = filter_ymd_candidate( - bestmatch, - DATESTRINGS_PATTERN, - copyear, - options.format, - options.min, - options.max, + result = _search_and_filter( + htmlstring, DATESTRINGS_PATTERN, DATESTRINGS_CATCH, copyear, options ) if result is not None: return result @@ -704,7 +687,6 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: lambda item: normalize_match(THREE_COMP_REGEX_B.match(item)), copyear, options, - incomplete=True, ) if result is not None: return result @@ -720,47 +702,34 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) if bestmatch is not None: - dateobject = datetime(int(bestmatch[1]), int(bestmatch[2]), 1) - if copyear == 0 or dateobject.year >= copyear: - result = validate_and_convert( - dateobject, options.format, earliest=options.min, latest=options.max - ) - if result is not None: - LOGGER.debug( - 'date found for pattern "%s": %s, %s', - YYYYMM_PATTERN, - bestmatch[1], - bestmatch[2], - ) - return result + result = _finalize_candidate( + datetime(int(bestmatch[1]), int(bestmatch[2]), 1), copyear, options + ) + if result is not None: + return result # 2 components, second option result = search_normalized( htmlstring, MMYYYY_PATTERN, - MMYYYY_YEAR, + SELECT_YMD_YEAR, normalize_two_comp, copyear, options, - incomplete=options.original, ) if result is not None: return result # try full-blown text regex on all HTML? - text_date = regex_parse(htmlstring) # todo: find all candidates and disambiguate? - if copyear == 0 or (text_date and text_date.year >= copyear): - result = validate_and_convert( - text_date, options.format, earliest=options.min, latest=options.max - ) - if result is not None: - return result + result = _finalize_candidate(regex_parse(htmlstring), copyear, options) + if result is not None: + return result # catchall: copyright mention if copyear != 0: LOGGER.debug("using copyright year as default") - dateobject = datetime(int(copyear), 1, 1) + dateobject = datetime(copyear, 1, 1) return dateobject.strftime(options.format) # last resort: 1 component @@ -773,17 +742,7 @@ def search_page(htmlstring: str, options: Extractor) -> str | None: options, ) if bestmatch is not None: - dateobject = datetime(int(bestmatch[0]), 1, 1) - if ( - is_valid_date( - dateobject, "%Y-%m-%d", earliest=options.min, latest=options.max - ) - and dateobject.year >= copyear - ): - LOGGER.debug( - 'date found for pattern "%s": %s', SIMPLE_PATTERN, bestmatch[0] - ) - return dateobject.strftime(options.format) + return _finalize_candidate(datetime(int(bestmatch[0]), 1, 1), copyear, options) return None @@ -901,26 +860,15 @@ def find_date( LOGGER.error("lxml cleaner error") # define expressions + text_content - if extensive_search: - date_expr = SLOW_PREPEND + DATE_EXPRESSIONS - else: - date_expr = FAST_PREPEND + DATE_EXPRESSIONS + date_expr = (SLOW_PREPEND if extensive_search else FAST_PREPEND) + DATE_EXPRESSIONS # then look for expressions # and try time elements - result = ( - examine_date_elements( - search_tree, - date_expr, - options, - ) - or examine_date_elements( - search_tree, - ".//title|.//h1", - options, - ) - or examine_time_elements(search_tree, options) - ) + result = examine_date_elements( + search_tree, + [date_expr, ".//title|.//h1"], + options, + ) or examine_time_elements(search_tree, options) if result is not None: return result diff --git a/htmldate/extractors.py b/htmldate/extractors.py index bd0c470c..b8f7a9ef 100644 --- a/htmldate/extractors.py +++ b/htmldate/extractors.py @@ -19,7 +19,7 @@ # own from .settings import CACHE_SIZE, MAX_SEGMENT_LEN -from .utils import Extractor, trim_text +from .utils import Extractor, remove_if_attached, trim_text from .validators import convert_date, correct_year, is_valid_date, validate_and_convert LOGGER = logging.getLogger(__name__) @@ -126,9 +126,6 @@ JSON_PUBLISHED = re.compile( rf'"datePublished": ?"({YEAR_RE}-{MONTH_RE}-{DAY_RE})', re.I ) -TIMESTAMP_PATTERN = re.compile( - rf"({YEAR_RE}-{MONTH_RE}-{DAY_RE}).[0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}" -) # English, French, German, Indonesian and Turkish dates cache MONTHS = [ @@ -176,45 +173,11 @@ re.I, ) -# core patterns -THREE_COMP_REGEX_A = re.compile(rf"({DAY_RE})[/.-]({MONTH_RE})[/.-]({YEAR_RE})") -THREE_COMP_REGEX_B = re.compile( - rf"({DAY_RE})/({MONTH_RE})/([0-9]{{2}})|({DAY_RE})[.-]({MONTH_RE})[.-]([0-9]{{2}})" -) -TWO_COMP_REGEX = re.compile(rf"({MONTH_RE})[/.-]({YEAR_RE})") - -# extensive search patterns -YEAR_PATTERN = re.compile(rf"^\D?({YEAR_RE})") -# bounded gap (\D{0,99}, not unbounded \D*) to avoid quadratic backtracking (ReDoS) -COPYRIGHT_PATTERN = re.compile( - rf"(?:©|\©|Copyright|\(c\))\D{{0,99}}(?:{YEAR_RE})?-?({YEAR_RE})\D" -) -THREE_PATTERN = re.compile(r"/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]") -THREE_CATCH = re.compile(r"([0-9]{4})/([0-9]{2})/([0-9]{2})") -THREE_LOOSE_PATTERN = re.compile(r"\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\D") -THREE_LOOSE_CATCH = re.compile(r"([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})") -SELECT_YMD_PATTERN = re.compile(r"\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\D") -SELECT_YMD_YEAR = re.compile(rf"({YEAR_RE})\D?$") -YMD_YEAR = re.compile(rf"^({YEAR_RE})") -DATESTRINGS_PATTERN = re.compile( - r"(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)" -) -DATESTRINGS_CATCH = re.compile(rf"({YEAR_RE})([01][0-9])([0-3][0-9])") -SLASHES_PATTERN = re.compile( - r"\D([0-3]?[0-9]/[01]?[0-9]/[0129][0-9]|[0-3][0-9]\.[01][0-9]\.[0129][0-9])\D" -) -SLASHES_YEAR = re.compile(r"([0-9]{2})$") -YYYYMM_PATTERN = re.compile(r"\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\D") -YYYYMM_CATCH = re.compile(rf"({YEAR_RE})[/.-](1[0-2]|0[1-9])") -MMYYYY_PATTERN = re.compile(r"\D([01]?[0-9][/.-][12][0-9]{3})\D") -MMYYYY_YEAR = re.compile(rf"({YEAR_RE})\D?$") -SIMPLE_PATTERN = re.compile(rf"(? HtmlElement: """Delete unwanted sections of an HTML document.""" for subtree in DISCARD_EXPRESSIONS(tree): - subtree.getparent().remove(subtree) + remove_if_attached(subtree) return tree @@ -242,6 +205,13 @@ def try_swap_values(day: int, month: int) -> tuple[int, int]: return (month, day) if month > 12 and day <= 12 else (day, month) +def _build_dmy(day: int, month: int, year: int) -> datetime: + "Build a datetime from day/month/year, fixing 2-digit years and day/month order." + year = correct_year(year) + day, month = try_swap_values(day, month) + return datetime(year, month, day) + + def regex_parse(string: str) -> datetime | None: """Try full-text parse for date elements using a series of regular expressions with particular emphasis on English, French, German and Turkish""" @@ -257,20 +227,22 @@ def regex_parse(string: str) -> datetime | None: ) # process and return try: - day, month, year = ( + dateobject = _build_dmy( int(match.group(groups[0])), - int(TEXT_MONTHS[match.group(groups[1]).lower().strip(".")]), + TEXT_MONTHS[match.group(groups[1]).lower()], int(match.group(groups[2])), ) - year = correct_year(year) - day, month = try_swap_values(day, month) - dateobject = datetime(year, month, day) - except ValueError: + except (KeyError, ValueError): return None LOGGER.debug("multilingual text found: %s", dateobject) return dateobject +def _parse_yyyymmdd(digits: str) -> datetime: + "Build a datetime from a leading 8-digit YYYYMMDD run." + return datetime(int(digits[:4]), int(digits[4:6]), int(digits[6:8])) + + def custom_parse( string: str, outputformat: str, min_date: datetime, max_date: datetime ) -> str | None: @@ -283,9 +255,7 @@ def custom_parse( # a. '201709011234' not covered by dateparser, and regex too slow if string[4:8].isdigit(): try: - candidate = datetime( - int(string[:4]), int(string[4:6]), int(string[6:8]) - ) + candidate = _parse_yyyymmdd(string) except ValueError: LOGGER.debug("8-digit error: %s", string[:8]) # return None # b. much faster than extensive parsing @@ -299,51 +269,50 @@ def custom_parse( except (OverflowError, TypeError, ValueError): LOGGER.debug("dateutil parsing error: %s", string) # c. plausibility test - if candidate is not None and ( - is_valid_date(candidate, outputformat, earliest=min_date, latest=max_date) - ): - LOGGER.debug("parsing result: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 2. Try YYYYMMDD, use regex match = YMD_NO_SEP_PATTERN.search(string) if match: try: - year, month, day = int(match[1][:4]), int(match[1][4:6]), int(match[1][6:8]) - candidate = datetime(year, month, day) + candidate = _parse_yyyymmdd(match[1]) except ValueError: LOGGER.debug("YYYYMMDD value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("YYYYMMDD match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 3. Try the very common YMD, Y-M-D, and D-M-Y patterns match = YMD_PATTERN.search(string) if match: try: if match.lastgroup == "day": - year, month, day = ( + candidate = datetime( int(match.group("year")), int(match.group("month")), int(match.group("day")), ) else: - day, month, year = ( + candidate = _build_dmy( int(match.group("day2")), int(match.group("month2")), int(match.group("year2")), ) - year = correct_year(year) - day, month = try_swap_values(day, month) - - candidate = datetime(year, month, day) except ValueError: # pragma: no cover LOGGER.debug("regex value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("regex match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 4. Try the Y-M and M-Y patterns match = YM_PATTERN.search(string) @@ -360,9 +329,11 @@ def custom_parse( except ValueError: LOGGER.debug("Y-M value error: %s", match[0]) else: - if is_valid_date(candidate, "%Y-%m-%d", earliest=min_date, latest=max_date): - LOGGER.debug("Y-M match: %s", candidate) - return candidate.strftime(outputformat) + result = validate_and_convert( + candidate, outputformat, earliest=min_date, latest=max_date + ) + if result is not None: + return result # 5. Try the other regex pattern dateobject = regex_parse(string) @@ -430,6 +401,13 @@ def try_date_expr( return None +def try_date_expr_opts(string: str | None, options: Extractor) -> str | None: + "Uncached options wrapper for try_date_expr (Extractor is unhashable)." + return try_date_expr( + string, options.format, options.extensive, options.min, options.max + ) + + def img_search( tree: HtmlElement, options: Extractor, @@ -491,9 +469,7 @@ def idiosyncrasies_search( if len(parts[0]) == 4: # year in first position candidate = datetime(int(parts[0]), int(parts[1]), int(parts[2])) else: # len(parts[2]) in (2, 4): # DD/MM/YY - day, month = try_swap_values(int(parts[0]), int(parts[1])) - year = correct_year(int(parts[2])) - candidate = datetime(year, month, day) + candidate = _build_dmy(int(parts[0]), int(parts[1]), int(parts[2])) return validate_and_convert( candidate, options.format, earliest=options.min, latest=options.max ) diff --git a/htmldate/meta.py b/htmldate/meta.py index 87a34a99..aefe5d8f 100644 --- a/htmldate/meta.py +++ b/htmldate/meta.py @@ -4,9 +4,8 @@ import logging -from .core import compare_reference from .extractors import try_date_expr -from .validators import filter_ymd_candidate, is_valid_date, is_valid_format +from .validators import _parse_and_validate, filter_ymd_candidate, is_valid_format LOGGER = logging.getLogger(__name__) @@ -24,9 +23,8 @@ def reset_caches() -> None: """Reset all known LRU caches used to speed-up processing. This may release some memory.""" # htmldate - compare_reference.cache_clear() + _parse_and_validate.cache_clear() filter_ymd_candidate.cache_clear() - is_valid_date.cache_clear() is_valid_format.cache_clear() try_date_expr.cache_clear() # charset_normalizer diff --git a/htmldate/utils.py b/htmldate/utils.py index 538a0cec..0cf8fbbf 100644 --- a/htmldate/utils.py +++ b/htmldate/utils.py @@ -127,14 +127,12 @@ def fetch_url(url: str) -> str | None: url: URL of the page to fetch. Returns: - HTML code as string, or Urllib3 response object (headers + body), or empty string in case - the result is invalid, or None if there was a problem with the network. + HTML code as a string, or None if the network request failed, the + response status was not 200, or the payload was invalid. """ # send try: - # read by streaming chunks (stream=True, iter_content=xx) - # so we can stop downloading as soon as MAX_FILE_SIZE is reached response = HTTP_POOL.request("GET", url, timeout=30) except Exception as err: LOGGER.error("download error: %s %s", url, err) # sys.exc_info()[0] @@ -150,7 +148,7 @@ def fetch_url(url: str) -> str | None: def is_dubious_html(beginning: str) -> bool: - "Assess if the object is proper HTML (awith a corresponding tag or declaration)." + "Assess if the object is proper HTML (with a corresponding tag or declaration)." return "html" not in beginning @@ -231,6 +229,13 @@ def load_html(htmlobject: bytes | str | HtmlElement) -> HtmlElement | None: return tree +def remove_if_attached(element: HtmlElement) -> None: + "Remove an element from its parent, if it still has one." + parent = element.getparent() + if parent is not None: + parent.remove(element) + + def clean_html(tree: HtmlElement, elemlist: list[str]) -> HtmlElement: "Delete selected elements." for element in tree.iter(elemlist): @@ -239,9 +244,7 @@ def clean_html(tree: HtmlElement, elemlist: list[str]) -> HtmlElement: try: element.drop_tree() except AttributeError: # pragma: no cover - parent = element.getparent() - if parent is not None: - parent.remove(element) + remove_if_attached(element) return tree diff --git a/htmldate/validators.py b/htmldate/validators.py index 2425f287..8fb96650 100644 --- a/htmldate/validators.py +++ b/htmldate/validators.py @@ -17,42 +17,56 @@ LOGGER.debug("minimum date setting: %s", MIN_DATE) -@lru_cache(maxsize=CACHE_SIZE) +def _is_in_range(dateobject: datetime, earliest: datetime, latest: datetime) -> bool: + """Check whether a datetime falls within the configured time window.""" + return ( + earliest.year <= dateobject.year <= latest.year + and earliest.timestamp() <= dateobject.timestamp() <= latest.timestamp() + ) + + def is_valid_date( date_input: datetime | str | None, outputformat: str, earliest: datetime, latest: datetime, ) -> bool: - """Validate a string w.r.t. the chosen outputformat and basic heuristics""" - # safety check + """Validate a date w.r.t. the chosen outputformat and time boundaries.""" if date_input is None: return False - # try if date can be parsed using chosen outputformat + # datetime path: no format needed for parsing, no cache required if isinstance(date_input, datetime): - dateobject = date_input - else: - # speed-up - try: - if outputformat == "%Y-%m-%d": - dateobject = datetime( - int(date_input[:4]), int(date_input[5:7]), int(date_input[8:10]) - ) - # default - else: - dateobject = datetime.strptime(date_input, outputformat) - except ValueError: - return False + result = _is_in_range(date_input, earliest, latest) + if not result: + LOGGER.debug("date not valid: %s", date_input) + return result - # year first, then full validation: not newer than today or stored variable - if ( - earliest.year <= dateobject.year <= latest.year - and earliest.timestamp() <= dateobject.timestamp() <= latest.timestamp() - ): - return True - LOGGER.debug("date not valid: %s", date_input) - return False + # string path: parse then validate (cached on the string + format combo) + return _parse_and_validate(date_input, outputformat, earliest, latest) + + +@lru_cache(maxsize=CACHE_SIZE) +def _parse_and_validate( + date_input: str, + outputformat: str, + earliest: datetime, + latest: datetime, +) -> bool: + """Parse a date string and validate it against time boundaries.""" + try: + if outputformat == "%Y-%m-%d": + # read the leading YYYY-MM-DD and ignore the rest + dateobject = datetime.fromisoformat(date_input[:10]) + else: + dateobject = datetime.strptime(date_input, outputformat) + except ValueError: + return False + + result = _is_in_range(dateobject, earliest, latest) + if not result: + LOGGER.debug("date not valid: %s", date_input) + return result def validate_and_convert( @@ -104,7 +118,6 @@ def plausible_year_filter( yearpat: re.Pattern[str], earliest: datetime, latest: datetime, - incomplete: bool = False, ) -> Counter[str]: """Filter the date patterns to find plausible years only""" occurrences = Counter(pattern.findall(htmlstring)) # slow! @@ -117,11 +130,9 @@ def plausible_year_filter( del occurrences[item] continue - lastdigits = year_match[1] - if not incomplete: - potential_year = int(lastdigits) - else: - potential_year = correct_year(int(lastdigits)) + # correct_year() is a no-op for already-4-digit years, so this also + # covers yearpat patterns that only ever capture 4 digits + potential_year = correct_year(int(year_match[1])) if not min_year <= potential_year <= max_year: LOGGER.debug("no potential year: %s", item) @@ -130,6 +141,13 @@ def plausible_year_filter( return occurrences +def update_reference(reference: int, candidate: int, original: bool) -> int: + "Fold a timestamp into the running reference: oldest if original, else newest." + if original: + return min(reference, candidate) if reference else candidate + return max(reference, candidate) + + def compare_values(reference: int, attempt: str, options: Extractor) -> int: """Compare the date expression to a reference""" try: @@ -137,17 +155,12 @@ def compare_values(reference: int, attempt: str, options: Extractor) -> int: except Exception as err: LOGGER.debug("datetime.strptime exception: %s for string %s", err, attempt) return reference - if options.original: - reference = min(reference, timestamp) if reference else timestamp - else: - reference = max(reference, timestamp) - return reference + return update_reference(reference, timestamp, options.original) @lru_cache(maxsize=CACHE_SIZE) def filter_ymd_candidate( - bestmatch: re.Match[str], - pattern: re.Pattern[str], + bestmatch: tuple[str, ...] | None, copyear: int, outputformat: str, min_date: datetime, @@ -155,11 +168,11 @@ def filter_ymd_candidate( ) -> str | None: """Filter free text candidates in the YMD format""" if bestmatch is not None: - pagedate = "-".join([bestmatch[1], bestmatch[2], bestmatch[3]]) + pagedate = "-".join(bestmatch[:3]) if is_valid_date(pagedate, "%Y-%m-%d", earliest=min_date, latest=max_date) and ( - copyear == 0 or int(bestmatch[1]) >= copyear + copyear == 0 or int(bestmatch[0]) >= copyear ): - LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate) + LOGGER.debug("date found: %s", pagedate) return convert_date(pagedate, "%Y-%m-%d", outputformat) return None @@ -169,10 +182,9 @@ def convert_date(datestring: str, inputformat: str, outputformat: str) -> str: # speed-up (%Y-%m-%d) if inputformat == outputformat: return datestring - # date object (speedup) + # datetime object passed directly (callers may violate the str annotation) if isinstance(datestring, datetime): return datestring.strftime(outputformat) - # normal dateobject = datetime.strptime(datestring, inputformat) return dateobject.strftime(outputformat) diff --git a/tests/evaluation.py b/tests/evaluation.py index 7282ab52..c80c802a 100644 --- a/tests/evaluation.py +++ b/tests/evaluation.py @@ -6,12 +6,8 @@ import os import re -try: - from cchardet import detect -except ImportError: - from charset_normalizer import detect - from htmldate import find_date +from htmldate.utils import decode_file from htmldate.validators import convert_date # Optional third-party libraries, only needed for the full benchmark @@ -43,31 +39,13 @@ def load_document(filename): """load mock page from samples""" - htmlstring = "" # look for the right directory for directory in ("test_set", "cache", "eval"): mypath = os.path.join(TEST_DIR, directory, filename) if os.path.isfile(mypath): break - # open and convert the file to str - try: - with open(mypath, "r", encoding="utf-8") as inputf: - htmlstring = inputf.read() - # encoding/windows fix for the tests - except UnicodeDecodeError: - # read as binary - with open(mypath, "rb") as inputf: - htmlbinary = inputf.read() - guessed_encoding = detect(htmlbinary)["encoding"] - if guessed_encoding: - try: - htmlstring = htmlbinary.decode(guessed_encoding) - except UnicodeDecodeError: - pass - if not htmlstring: - # print("Encoding error, using binary:", mypath) - htmlstring = htmlbinary - return htmlstring + with open(mypath, "rb") as inputf: + return decode_file(inputf.read()) def run_htmldate_extensive(htmlstring): diff --git a/tests/realworld_tests.py b/tests/realworld_tests.py index 0e0ee129..7f1217ab 100644 --- a/tests/realworld_tests.py +++ b/tests/realworld_tests.py @@ -100,8 +100,6 @@ "https://www.mozilla.org/en-US/security/advisories/mfsa2024-17/": "mozilla.org.mfsa2024-17.html", } -MEDIACLOUD_PAGES = {"pagename": "thing.html"} - def load_mock_page(url, dirname="cache", dictname=MOCK_PAGES): """load mock page from samples""" @@ -109,11 +107,6 @@ def load_mock_page(url, dirname="cache", dictname=MOCK_PAGES): return inputf.read() -def load_mediacloud_page(url): - """load mediacloud page from samples""" - return load_mock_page(url, dirname="test_set", dictname=MEDIACLOUD_PAGES) - - def test_no_date(): "These pages should not return any date" assert ( @@ -765,7 +758,7 @@ def test_dependencies(): def test_cli(): "Test the command-line interface" - # third test: Linux and MacOS only + # Linux and MacOS only if os.name != "nt": testargs = [] args = parse_args(testargs) @@ -779,16 +772,3 @@ def test_cli(): process_args(args) assert f.getvalue() == "2017-07-12\n" sys.stdin = sys.__stdin__ - - -if __name__ == "__main__": - # meta - test_readme_examples() - test_dependencies() - - # module-level - test_no_date() - test_exact_date() - test_approximate_date() - test_search_html() - test_cli() diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 5785084c..14247585 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -1,4 +1,3 @@ -# pylint:disable-msg=W1401 """ Unit tests for the htmldate library. """ @@ -24,6 +23,7 @@ from htmldate.core import ( compare_reference, examine_date_elements, + examine_text, find_date, search_page, search_pattern, @@ -181,6 +181,12 @@ def test_input(): assert get_max_date("2020-02-20T13:30:00") == datetime.datetime(2020, 2, 20, 13, 30) +def test_examine_text(): + """test early-exit guards in examine_text""" + assert examine_text(" ab ", OPTIONS) is None + assert examine_text("a b c", OPTIONS) is None + + def test_sanity(): """Test if function arguments are interpreted and processed correctly.""" # XPath looking for date elements @@ -284,12 +290,10 @@ def test_exact_date(): ) == "2017-01-09" ) - assert ( - find_date( - '' - ) - == "2017-01-01" - ) + # copyrightyear reserve, with and without extensive search + htmldoc = '' + for extensive in (True, False): + assert find_date(htmldoc, extensive_search=extensive) == "2017-01-01" # original date htmldoc = '' @@ -517,6 +521,10 @@ def test_exact_date(): ) == "2011-09-28" ) + # a shortcut