# reranker calibration probe (#227)
date:       2026-08-30T19:21:20+0300
host:       spark-f8a9
url:        http://localhost:8001
model:      Qwen/Qwen3-Reranker-0.6B
git sha:    b867897
container:  vllm/vllm-openai@sha256:8bd082c274fae025b7079498fe1da65182ba1d4c2188c0f5a68c1042c38c3695
args:       ["/usr/local/bin/mg-logwrap","vllm","serve","Qwen/Qwen3-Reranker-0.6B","--served-model-name=Qwen/Qwen3-Reranker-0.6B","--host=0.0.0.0","--port=8000","--runner=pooling","--convert=classify","--attention-config={\"backend\": \"auto\"}","--no-enforce-eager","--hf-overrides={\"architectures\": [\"Qwen3ForSequenceClassification\"], \"classifier_from_token\": [\"no\", \"yes\"], \"is_original_qwen3_reranker\": true}","--max-model-len=8192","--gpu-memory-utilization=0.06"]
templated:  False
vllm:       0.26.1rc1.dev942+g5a4c8d992

## probe set — relevance_score per document, usage.prompt_tokens

[sky]  Q: What colour is the sky on a clear day?
  prompt_tokens total=67 per_pair=22.3  latency=219ms
  [0] 0.973  On a clear day the sky is blue because air scatters short wavelengths.
  [1] 0.794  Cats purr when they are content.
  [2] 0.290  The invoice is due on the last business day of the month.
  ranking: [0, 1, 2]

[ports-ledger]  Q: Which file lists the ports the gateway exposes?
  prompt_tokens total=71 per_pair=23.7  latency=62ms
  [0] 0.936  The gateway port ledger is docs/gateway-fleet.md, which lists 8000 and 8001.
  [1] 0.524  Cats purr when they are content.
  [2] 0.510  Bananas are a good source of potassium.
  ranking: [0, 1, 2]

[france (assess probe)]  Q: What is the capital of France?
  prompt_tokens total=51 per_pair=17.0  latency=118ms
  [0] 0.973  Paris is the capital and most populous city of France.
  [1] 0.216  The Amazon rainforest spans several South American countries.
  [2] 0.876  Bananas are a good source of potassium.
  ranking: [0, 2, 1]

[toolbatch inversion]  Q: How does the tool batcher group tool calls into one request?
  prompt_tokens total=81 per_pair=27.0  latency=73ms
  [0] 0.790  NOTICE: this file is generated; do not edit by hand.
  [1] 0.988  toolbatch collects consecutive tool calls and issues them as a single batched 
  [2] 0.741  Cats purr when they are content.
  ranking: [1, 0, 2]

[graded relevance]  Q: How do I stop the lobes fleet without deleting the deployment directory?
  prompt_tokens total=89 per_pair=29.7  latency=115ms
  [0] 0.970  Run `lobes stop --apply`; it runs `docker compose down`, removing the containe
  [1] 0.700  `lobes stop` stops the fleet.
  [2] 0.283  Cats purr when they are content.
  ranking: [0, 1, 2]

## instruction probe — same pair with and without a top-level `instruction`
  /v1/rerank: without=[(0, 0.9358), (1, 0.5238)] tokens=53
  /v1/rerank: with   =[(0, 0.9358), (1, 0.5238)] tokens=53
  /v1/rerank: instruction changes scores: False
  /v1/score: without=[(0, 0.9358), (1, 0.5238)] tokens=53
  /v1/score: with   =[(0, 0.9358), (1, 0.5238)] tokens=53
  /v1/score: instruction changes scores: False

## latency — 1 query x 5 docs, median of 5 (1 warm-up discarded)
  samples_ms=[64.6, 28.0, 27.5, 28.3, 17.1]
  median_ms=28.0
