% turboquant-pro: A Multi-Surface Compression System -- IEEE AIoT 2026 (Track 3)
% IEEE conference template (IEEEtran). Compile with pdflatex (IEEEtran.cls from texlive/Overleaf).
%
% STATUS: skeleton with REAL established numbers + the analytical memory budget
% (benchmarks/benchmark_edge.py). Cells marked \tbd{} are on-device measurements
% to fill before submission, on the Jetson Nano (4GB) and a Volta GPU:
%   memory budget: python benchmarks/benchmark_edge.py --energy --json out/edge_<device>.json
%   end-to-end:    python -m benchmarks.benchmark_e2e --model <gguf> --quantizer <prov> --energy
% IEEE Computer Society Proceedings format (double column, 10pt, letter) per the
% IEEE AIoT 2026 CFP -- IEEEtran 'compsoc' conference mode.
\documentclass[10pt,conference,compsoc]{IEEEtran}
\IEEEoverridecommandlockouts

\usepackage{cite}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{textcomp}
\usepackage{xcolor}
\usepackage[hidelinks]{hyperref}
\usepackage{url}
\usepackage{orcidlink}

% Placeholder macro for numbers that must come from on-device runs.
\newcommand{\tbd}[1][TBD]{\textcolor{red}{\textbf{[#1]}}}
\setlength{\tabcolsep}{4pt}  % tighter columns to fit the single-column measure

\begin{document}

\title{\texttt{turboquant-pro}: A Multi-Surface Compression System for\\
Foundation-Model Inference on Resource-Constrained AIoT Devices}

\author{
\IEEEauthorblockN{Andrew H. Bond~\orcidlink{0009-0003-2599-6158}, Senior Member, IEEE}
\IEEEauthorblockA{Department of Computer Engineering\\
San Jose State University, San Jose, CA, USA\\
andrew.bond@sjsu.edu}
}

\maketitle

\begin{abstract}
Deploying large language and embedding models on AIoT edge hardware is
constrained less by compute than by \emph{memory and energy}: model weights and
the autoregressive key--value (KV) cache exceed the budgets of consumer GPUs,
edge accelerators, and CPUs. We present \texttt{turboquant-pro}, an open-source
\emph{system} that compresses the three surfaces dominating edge memory---model
weights, the RoPE-aware LLM KV cache, and the embedding store used for on-device
retrieval---within one deployable pipeline. The pipeline composes established
primitives: the TurboQuant rotate-and-scalar-quantize algorithm~\cite{turboquant}
and Matryoshka representations~\cite{mrl}, which we extend with learned codebooks,
RoPE-aware KV handling, bit-packing, and a device-tiered \texttt{AutoConfig}. Our
contribution is the \emph{integration} and its edge deployment, not the underlying
quantizer. On production embeddings the system attains up to $27\times$
compression at $99.8\%$ recall@10 (with oversampling and reranking) and learned
codebooks cut quantization error by $22\%$; at 3-bit, KV memory shrinks
$5.3\times$ versus fp16---shrinking a 7B model's weights-plus-KV footprint from
14\,GB (fp16) to 2.7\,GB, within the nominal budget of a 4\,GB device. We evaluate
end-to-end deployment on a Jetson Nano (4\,GB, 10\,W) and a Volta-class GPU,
reporting peak memory, throughput, and \emph{energy-per-token}, and emphasize
task-relevant metrics: cosine similarity is \emph{not} a reliable predictor of
retrieval quality under aggressive compression. \texttt{turboquant-pro} is
open-source (PyPI; Zenodo DOI~10.5281/zenodo.20660087)~\cite{turboquantpro} with
397 tests, and runs on Volta-class GPUs and CPU.
\end{abstract}

\begin{IEEEkeywords}
edge AI, AIoT, LLM deployment, model compression, KV cache, quantization,
energy efficiency, on-device inference
\end{IEEEkeywords}

\section{Introduction}
Foundation models are moving onto AIoT devices---smart-home hubs, industrial
gateways, robots, and field sensors---for privacy, latency, and offline
operation. On such IoT-class hardware (often a few watts and a few gigabytes of
memory shared with the OS) the binding constraint is \emph{memory and energy},
not arithmetic; and the data is increasingly multi-modal---vision, audio, and
text from on-board sensors---so models and their embedding stores must be
compressed \emph{together}, on-device. Three memory sinks dominate: the model
\emph{weights}; the
\emph{KV cache}, which grows with context length $\times$ layers and quickly
dwarfs the weights at long context; and, for retrieval-augmented edge agents,
the \emph{embedding/index store}. Prior compression work typically optimizes one
surface in isolation and reports proxy metrics.

This paper contributes a \emph{system}, not a new quantizer. \texttt{turboquant-pro}
provides: (1) a single pipeline that applies low-bit quantization across all three
edge memory surfaces---weights, KV cache, and embeddings; (2) a RoPE-aware KV-cache
compressor for long-context on-device inference; (3) a device-tiered
\texttt{AutoConfig} deployment path for Volta+/CPU targets; (4) a batched SIMD
asymmetric-distance kernel that searches the compressed embedding store on the CPU
at $7.9\times$ the speed of decompress-and-scan ($3683$ qps, recall@10 $0.9995$);
and (5) an edge
evaluation on a 10\,W Jetson Nano reporting memory, throughput, and
energy-per-token, with the explicit caveat that cosine similarity is not a reliable
proxy for task quality. The core scalar quantizer is TurboQuant~\cite{turboquant}
(Zandieh \emph{et al.}); Matryoshka truncation~\cite{mrl} and Lloyd--Max
quantization are likewise prior art. What is new is their composition into one
auditable, deployable system and its measurement on IoT-class hardware.

\section{Background and Related Work}
\textbf{Weight quantization.} GPTQ~\cite{gptq} and AWQ~\cite{awq} push LLM weights
to 3--4 bits with small accuracy loss. \textbf{KV-cache quantization.}
KVQuant~\cite{kvquant} and KIVI~\cite{kivi} compress the cache that dominates
long-context memory. \textbf{Matryoshka representations}~\cite{mrl} enable
truncatable embeddings. \textbf{Vector quantization} underpins approximate
nearest-neighbor search~\cite{faiss}. The gap: edge deployment needs a
\emph{whole-system} memory budget across all three surfaces, evaluated on
task-grounded rather than proxy metrics.

\section{The \texttt{turboquant-pro} System}
\texttt{turboquant-pro} applies one pipeline to each memory surface
(Fig.~\ref{fig:pipeline}): (i) PCA--Matryoshka~\cite{mrl} rotate-and-truncate;
(ii) \emph{TurboQuant}~\cite{turboquant}---a random orthogonal rotation followed
by near-optimal scalar quantization---which is the core quantizer we build on;
(iii) learned codebooks that refine the scalar levels ($-22\%$ error vs.\
uniform); and (iv) bit-packing (e.g., $8\times3$-bit $\rightarrow$ 3 bytes) with
the L2 norm preserved alongside the packed bits. For the KV cache, a RoPE-aware
quantizer and a full-precision hot-window of recent tokens preserve attention
fidelity; for retrieval, a compressed HNSW index supports on-device search.
Scalar quantization plus a cheap rotation (no large codebook tables) is well
suited to CPU and edge accelerators. The novelty is the \emph{integration}---one
quantizer applied coherently across weights, KV, and embeddings with per-device
configuration---not the quantizer itself.

\begin{figure}[t]
\centering
% \includegraphics[width=\columnwidth]{figs/pipeline.pdf}  % TODO: export from poster-assets
\fbox{\parbox{0.9\columnwidth}{\centering\vspace{1.2em} Raw $\rightarrow$
PCA--Matryoshka $\rightarrow$ orthogonal rotation $\rightarrow$ Lloyd--Max
quantize $\rightarrow$ bit-pack \vspace{1.2em}}}
\caption{The \texttt{turboquant-pro} pipeline (TurboQuant scalar quantization at
its core), applied to weights, KV cache, and embeddings.}
\label{fig:pipeline}
\end{figure}

\noindent\textbf{From algorithm to deployable system.} Beyond the core pipeline,
\texttt{turboquant-pro} ships the integrations and production tooling that make it
a \emph{deployable} AIoT system rather than a reference implementation
(Table~\ref{tab:caps}), backed by 397 tests. Three capabilities stand out. The
\textbf{pgvector} integration stores \emph{and searches} compressed embeddings
inside PostgreSQL without decompressing them, turning a standard database into an
on-device retrieval store. \textbf{Learned codebooks} (\texttt{fit\_codebook})
fit the scalar levels to the empirical
data distribution rather than assuming a Gaussian, raising embedding cosine
similarity from $0.978$ to over $0.99$ at the same bit-width. A \textbf{quality
monitor} adds production observability---rolling-window cosine tracking,
Kolmogorov--Smirnov drift detection, alert callbacks, and Prometheus-compatible
metrics---so compression-quality degradation is caught in the field. These are
shipped capabilities; the evaluation in this paper focuses on the compression and
edge-memory budget.

\begin{table}[t]
\caption{\texttt{turboquant-pro} capabilities and their role in edge/AIoT
deployment. The paper evaluates the compression core; the rest is what makes it
deployable.}
\label{tab:caps}
\centering
\footnotesize
\begin{tabular}{p{2.45cm}p{4.95cm}}
\toprule
Capability & Role in edge/AIoT deployment \\
\midrule
\textbf{pgvector-native} compression & store and similarity-search compressed
embeddings in PostgreSQL \emph{without decompressing} \\
\textbf{Learned codebooks} (\texttt{fit\_codebook}) & data-fitted Lloyd--Max
levels; cosine $0.978\!\to\!0.99{+}$ at equal bits \\
\textbf{Multi-modal} presets & text (BGE-M3, E5, ada-002), vision (CLIP, SigLIP),
audio (Whisper), code (CodeBERT, CodeLlama); per-encoder PCA $+$ bit-width \\
Model-aware $+$ device-adaptive config & \texttt{from\_pretrained} (Llama-3-8B/70B
presets), autotune, hardware detection \\
\textbf{Observability} & \texttt{QualityMonitor}: KS-test drift detection,
Prometheus metrics, alert callbacks \\
Transport codec (NATS) & compresses embeddings/telemetry over edge--cloud buses \\
Serving \& vector-DB connectors & vLLM; FAISS/HNSW; Milvus, Pinecone, Qdrant,
Weaviate \\
Runtime \& coding & CPU SIMD ADC search (AVX2/NEON), Volta+ GPU, CuPy; ANS entropy coding $+$
bit-packing \\
\bottomrule
\end{tabular}
\end{table}

\section{Edge Deployment Design}
We model the device memory budget as
$M_{\text{dev}} \ge M_{\text{weights}} + M_{\text{KV}}(L, B) + M_{\text{index}}$,
where $L$ is context length and $B$ batch. \texttt{turboquant-pro} reduces each
term: weights via low-bit packing, KV via the RoPE-aware compressor (linear in
layers and context), and the index via PCA-truncation plus scalar quantization.
\texttt{AutoConfig.from\_pretrained} selects a bit-width per device tier (Volta+
GPU $\rightarrow$ edge accelerator $\rightarrow$ CPU-only).

\section{Experimental Setup}
\textbf{Devices.} A Jetson Nano (Tegra X1, Maxwell, 4\,GB shared, 10\,W)---our
primary IoT-class target, energy measured via \texttt{tegrastats}---and a
Volta-class GPU (NVML-measured energy). Memory budgets for Jetson Orin and
Raspberry~Pi~5 are reported analytically (Table~\ref{tab:fit}).
\textbf{Models.} Llama-3.2-1B/3B, Qwen2.5-7B, Mistral-7B.
\textbf{Tasks.} a retrieval set (recall@10) and a generation task
(task accuracy / perplexity). \textbf{Baselines.} fp16, GPTQ, AWQ, and KV-cache
quantization. \textbf{Metrics.} peak memory, throughput (tok/s), and
\emph{energy-per-token} (J/tok), the last measured via on-board power telemetry
(NVML on GPU; \texttt{tegrastats} on Jetson). Memory budgets below are analytical
(2\,B/elem fp16 vs.\ bit-packed at the target width) and verified against the
library's measured footprint; throughput and energy are measured on device.

\section{Results}

\subsection{Memory budget and device fit}
Table~\ref{tab:budget} reports the weights-plus-KV budget at 8K context. At
3-bit, KV shrinks $5.3\times$ and the total footprint $\approx 5\times$ (a 7B
model goes from 14.2\,GB to 2.7\,GB). The practical consequence
(Table~\ref{tab:fit}): the 7B fp16 footprint exceeds common edge budgets
(4--12\,GB), whereas every tested model fits under \texttt{turboquant-pro}. These
are \emph{nominal} device capacities; usable RAM is lower (OS and runtime
overhead), so Table~\ref{tab:device} measures only the models we actually run on
each device.

\begin{table}[t]
\caption{Memory footprint (GB), weights $+$ KV @ 8K context (analytical).}
\label{tab:budget}
\centering
\footnotesize
\begin{tabular}{lrrrrr}
\toprule
Model & Params & W fp16 & W 3-bit & KV fp16 & Total 3-bit \\
\midrule
Llama-3.2-1B & 1.24B & 2.31 & 0.43 & 0.25 & 0.48 \\
Llama-3.2-3B & 3.21B & 5.98 & 1.12 & 0.88 & 1.28 \\
Qwen2.5-7B   & 7.6B  & 14.16 & 2.66 & 0.44 & 2.74 \\
\bottomrule
\end{tabular}
\end{table}

\begin{table}[t]
\caption{Models that fit a device's \emph{nominal} memory: fp16 vs.\
\texttt{tq-pro} (\texttt{turboquant-pro} 3-bit), weights $+$ KV @ 8K context.}
\label{tab:fit}
\centering
\begin{tabular}{lrcc}
\toprule
Device tier & Budget & fp16 & \texttt{tq-pro} \\
\midrule
Jetson Nano (ours) & 4\,GB & 1/3 & \textbf{3/3} \\
Jetson Orin Nano & 8\,GB  & 2/3 & \textbf{3/3} \\
Jetson Orin NX   & 16\,GB & 3/3 & 3/3 \\
Raspberry Pi 5   & 8\,GB  & 2/3 & \textbf{3/3} \\
Consumer GPU     & 12\,GB & 2/3 & \textbf{3/3} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{On-device throughput and energy}
Table~\ref{tab:device} reports measured end-to-end decode throughput and
energy-per-token on each target. \emph{(To be filled from
\texttt{benchmark\_e2e.py --energy} runs on hardware; numbers below are
placeholders. The Quantizer column states the provenance of the running
weights---e.g.\ \texttt{gguf-q3\_k} baseline vs.\ \texttt{turboquant-pro}
export---so the J/tok figures are unambiguous.)}

\begin{table}[t]
\caption{Measured end-to-end decode throughput and energy-per-token, 3-bit class.}
\label{tab:device}
\centering
\begin{tabular}{lllrr}
\toprule
Device & Model & Quantizer & tok/s & J/tok \\
\midrule
Jetson Nano (4\,GB) & Llama-3.2-1B & \tbd & \tbd & \tbd \\
Jetson Nano (4\,GB) & Llama-3.2-3B & \tbd & \tbd & \tbd \\
Volta GPU           & Qwen2.5-7B   & \tbd & \tbd & \tbd \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Embedding compression and retrieval quality}
For the on-device retrieval surface, \texttt{turboquant-pro} achieves up to
$27\times$ embedding compression at $99.8\%$ recall@10 with $5\times$ oversampling
and reranking, benchmarked identically across methods on 50K production
embeddings~\cite{turboquantpro}. Crucially, cosine similarity to the original vector
is \emph{not} a reliable proxy: PCA-256+TQ3 has lower cosine (0.963) yet higher
recall@10 (78.2\%) than PCA-384+TQ3 (0.979 cosine, 76.4\% recall). We therefore
report task-relevant retrieval metrics throughout.

\textbf{Fast on-device search.} Edge retrieval must run on the CPU, without a
server-class GPU. We provide a batched asymmetric-distance (ADC) kernel that scans
the packed codes directly---tabulating per-query lookup tables and accumulating
$\sum_j \mathrm{LUT}[j,\mathrm{code}[j]]$ with an 8-bit SIMD table lookup (x86
\texttt{pshufb}; the same scheme maps to ARM NEON \texttt{vqtbl} on Jetson-class
parts). On an x86 core it reaches \textbf{3683 qps at recall@10 $0.9995$}
(96\,B/vector), a $7.9\times$ speedup over decompress-and-scan, with a pure-NumPy
fallback when the kernel is not compiled. This keeps the compressed embedding store
searchable at interactive latency on edge CPUs, not just GPUs.

\section{Discussion and Limitations}
The measured KV footprint carries fixed overhead (the full-precision hot window,
codebooks) that amortizes only at long context; for very short contexts,
uncompressed caching can be smaller. The smallest devices still cannot host the
largest models even compressed. Reranking adds compute at the edge that must be
budgeted against its retrieval-quality gain. Finally, the throughput reported by
the KV-path microbenchmark is not end-to-end model latency; full-model numbers
(Table~\ref{tab:device}) come from device runs with the model runtime.

\section{Conclusion}
\texttt{turboquant-pro} compresses the three memory surfaces that block
foundation-model inference at the edge, turning models that overflow IoT-class
budgets into ones that fit---with task-grounded quality and on-device energy
accounting. The contribution is the integrated, deployable system around an
existing near-optimal quantizer, not the quantizer itself. Code, tests, and
benchmarks (including \texttt{benchmark\_edge.py} and \texttt{benchmark\_e2e.py})
are open-source.

\section*{Reproducibility}
\texttt{turboquant-pro}: PyPI; DOI~10.5281/zenodo.20660087. The memory-budget
numbers are produced by \texttt{benchmarks/benchmark\_edge.py}; the end-to-end
throughput/energy by \texttt{benchmarks/benchmark\_e2e.py}.

\begin{thebibliography}{00}
\bibitem{gptq} E. Frantar, S. Ashkboos, T. Hoefler, and D. Alistarh, ``GPTQ:
Accurate post-training quantization for generative pre-trained transformers,'' in
\emph{Proc. Int. Conf. Learn. Represent. (ICLR)}, 2023. arXiv:2210.17323.
\bibitem{awq} J. Lin \emph{et al.}, ``AWQ: Activation-aware weight quantization
for on-device LLM compression and acceleration,'' in \emph{Proc. Mach. Learn.
Syst. (MLSys)}, 2024. arXiv:2306.00978.
\bibitem{kvquant} C. Hooper \emph{et al.}, ``KVQuant: Towards 10 million context
length LLM inference with KV cache quantization,'' in \emph{Proc. Adv. Neural
Inf. Process. Syst. (NeurIPS)}, 2024. arXiv:2401.18079.
\bibitem{kivi} Z. Liu \emph{et al.}, ``KIVI: A tuning-free asymmetric 2-bit
quantization for KV cache,'' in \emph{Proc. Int. Conf. Mach. Learn. (ICML)},
2024. arXiv:2402.02750.
\bibitem{mrl} A. Kusupati \emph{et al.}, ``Matryoshka representation learning,''
in \emph{Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)}, 2022. arXiv:2205.13147.
\bibitem{faiss} J. Johnson, M. Douze, and H. J\'egou, ``Billion-scale similarity
search with GPUs,'' \emph{IEEE Trans. Big Data}, vol. 7, no. 3, pp. 535--547,
2021, doi:10.1109/TBDATA.2019.2921572.
\bibitem{turboquant} A. Zandieh, M. Daliri, M. Hadian, and V. Mirrokni,
``TurboQuant: Online vector quantization with near-optimal distortion rate,''
2025. arXiv:2504.19874.
\bibitem{turboquantpro} A. H. Bond, ``\texttt{turboquant-pro}: Multi-surface
model compression for edge inference,'' Zenodo, 2026,
doi:10.5281/zenodo.20660087.
\end{thebibliography}

\end{document}
