Skip to content

[CI][Bugfix] Fix bad tolerance for test_batch_base64_embedding #16221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 9, 2025
Merged

[CI][Bugfix] Fix bad tolerance for test_batch_base64_embedding #16221

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0834984
Fix bad tolerance for test_batch_base64_embedding
mgoin Apr 8, 2025
b49b62b
Remove unneeded change
mgoin Apr 8, 2025
f8cc642
Add larger tolerance
mgoin Apr 8, 2025
3e3b360
Use check_embeddings_close
mgoin Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions tests/entrypoints/openai/test_embedding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from vllm.entrypoints.openai.protocol import EmbeddingResponse
from vllm.transformers_utils.tokenizer import get_tokenizer

from ...models.embedding.utils import check_embeddings_close
from ...utils import RemoteOpenAIServer

MODEL_NAME = "intfloat/multilingual-e5-small"
Expand Down Expand Up @@ -190,30 +191,35 @@ async def test_batch_base64_embedding(client: openai.AsyncOpenAI,
responses_float = await client.embeddings.create(input=input_texts,
model=model_name,
encoding_format="float")
float_data = [d.embedding for d in responses_float.data]

responses_base64 = await client.embeddings.create(input=input_texts,
model=model_name,
encoding_format="base64")

decoded_responses_base64_data = []
base64_data = []
for data in responses_base64.data:
decoded_responses_base64_data.append(
base64_data.append(
np.frombuffer(base64.b64decode(data.embedding),
dtype="float32").tolist())

assert responses_float.data[0].embedding == decoded_responses_base64_data[
0]
assert responses_float.data[1].embedding == decoded_responses_base64_data[
1]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=base64_data,
name_0="float",
name_1="base64",
)

# Default response is float32 decoded from base64 by OpenAI Client
responses_default = await client.embeddings.create(input=input_texts,
model=model_name)
default_data = [d.embedding for d in responses_default.data]

assert responses_float.data[0].embedding == responses_default.data[
0].embedding
assert responses_float.data[1].embedding == responses_default.data[
1].embedding
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=default_data,
name_0="float",
name_1="default",
)


@pytest.mark.asyncio
Expand Down