Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions libs/core/langchain_core/language_models/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import importlib.util
import warnings
from abc import ABC, abstractmethod
from collections.abc import Callable, Mapping, Sequence
Expand Down Expand Up @@ -38,12 +39,7 @@
if TYPE_CHECKING:
from langchain_core.outputs import LLMResult

try:
from transformers import GPT2TokenizerFast # type: ignore[import-not-found]

_HAS_TRANSFORMERS = True
except ImportError:
_HAS_TRANSFORMERS = False
_HAS_TRANSFORMERS = importlib.util.find_spec("transformers") is not None


class LangSmithParams(TypedDict, total=False):
Expand Down Expand Up @@ -93,6 +89,11 @@ def get_tokenizer() -> Any:
"Please install it with `pip install transformers`."
)
raise ImportError(msg)

from transformers import ( # type: ignore[import-not-found] # noqa: PLC0415
GPT2TokenizerFast,
)

# create a GPT-2 tokenizer instance
return GPT2TokenizerFast.from_pretrained("gpt2")

Expand Down
10 changes: 4 additions & 6 deletions libs/core/langchain_core/vectorstores/in_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import importlib.util
import json
import uuid
from pathlib import Path
Expand All @@ -23,12 +24,7 @@

from langchain_core.embeddings import Embeddings

try:
import numpy as np

_HAS_NUMPY = True
except ImportError:
_HAS_NUMPY = False
_HAS_NUMPY = importlib.util.find_spec("numpy") is not None


class InMemoryVectorStore(VectorStore):
Expand Down Expand Up @@ -439,6 +435,8 @@ def max_marginal_relevance_search_by_vector(
)
raise ImportError(msg)

import numpy as np # noqa: PLC0415

mmr_chosen_indices = maximal_marginal_relevance(
np.array(embedding, dtype=np.float32),
[vector for _, _, vector in prefetch_hits],
Expand Down
24 changes: 12 additions & 12 deletions libs/core/langchain_core/vectorstores/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,21 @@

from __future__ import annotations

import importlib.util
import logging
import warnings
from typing import TYPE_CHECKING, cast

try:
import numpy as np

_HAS_NUMPY = True
except ImportError:
_HAS_NUMPY = False

try:
import simsimd as simd # type: ignore[import-not-found]
_HAS_NUMPY = importlib.util.find_spec("numpy") is not None
_HAS_SIMSIMD = importlib.util.find_spec("simsimd") is not None

_HAS_SIMSIMD = True
except ImportError:
_HAS_SIMSIMD = False

if TYPE_CHECKING:
import numpy as np

Matrix = list[list[float]] | list[np.ndarray] | np.ndarray


logger = logging.getLogger(__name__)


Expand All @@ -54,6 +48,8 @@ def _cosine_similarity(x: Matrix, y: Matrix) -> np.ndarray:
)
raise ImportError(msg)

import numpy as np # noqa: PLC0415

if len(x) == 0 or len(y) == 0:
return np.array([[]])

Expand Down Expand Up @@ -98,6 +94,8 @@ def _cosine_similarity(x: Matrix, y: Matrix) -> np.ndarray:
similarity[np.isnan(similarity) | np.isinf(similarity)] = 0.0
return cast("np.ndarray", similarity)

import simsimd as simd # type: ignore[import-not-found] # noqa: PLC0415

x = np.array(x, dtype=np.float32)
y = np.array(y, dtype=np.float32)
return 1 - np.array(simd.cdist(x, y, metric="cosine"))
Expand Down Expand Up @@ -130,6 +128,8 @@ def maximal_marginal_relevance(
)
raise ImportError(msg)

import numpy as np # noqa: PLC0415

if min(k, len(embedding_list)) <= 0:
return []
if query_embedding.ndim == 1:
Expand Down