02 — Backbone Comparison
Compare all five backbones and auto-selection with candidates.
The pretrained backbone is where Vyntri's features come from. Bigger backbones usually transfer better, but cost more to extract on CPU. This tutorial compares all five registered backbones and shows backbone="auto" with candidate control.
Setup
from pathlib import Path
import numpy as np
from PIL import Image
from vyntri import Vyntri
from vyntri.data import splitCreate a dataset where backbones differ
Solid colors are too easy — every backbone scores 100%. This dataset uses per-class shifted Gaussian noise so the choice of backbone shows a real effect.
def make_noise(root, classes, per_class, seed=0):
root = Path(root)
rng = np.random.default_rng(seed)
for i, name in enumerate(classes):
folder = root / name
folder.mkdir(parents=True, exist_ok=True)
shift = i * 0.7
for j in range(per_class):
img = (rng.normal(shift, 1.0, (32, 32, 3)) * 40 + 128).clip(0, 255)
Image.fromarray(img.astype(np.uint8)).save(folder / f'{name}_{j:02d}.png')
return root
DATA = Path('backbone_data')
train = make_noise(DATA / 'train', ['cats', 'dogs', 'birds'], per_class=20, seed=1)
test = make_noise(DATA / 'test', ['cats', 'dogs', 'birds'], per_class=12, seed=99)
print('ready:', train)Fit each backbone and report
s = split(str(train), train=0.7, val=0.1, test=0.2, seed=42)
def fit_and_report(backbone):
model = Vyntri(backbone=backbone, seed=42)
model.fit(train=s.train, val=s.val, progress=False)
timings = model.metadata_['timings']
test_acc = model.evaluate(s.test).accuracy
print(f'{backbone:<20} val={model.validation_accuracy_:.3f} '
f'test={test_acc:.3f} '
f'extract={timings["feature_extraction_s"]:.2f}s '
f'adapt={timings["adaptation_s"]:.2f}s '
f'total={timings["total_s"]:.2f}s')
return model
models = {}
for backbone in ['mobilenet_v3_small', 'resnet18', 'resnet50', 'efficientnet_b0', 'convnext_tiny']:
models[backbone] = fit_and_report(backbone)Auto backbone selection
Vyntri can pick the best backbone for your data: every candidate is scored with LogME, the top candidates are validated with the analytic pipeline, and the winner is selected.
auto = Vyntri(backbone='auto', seed=42)
auto.fit(train=s.train, val=s.val, progress=False)
sel = auto.metadata_['selection']
print(f"selected: {sel['selected']} (validation {sel['selected_accuracy']:.3f})")
print(f"ranked: {sel['ranked']}")
print(f"scores: { {k: round(v, 3) for k, v in sel['scores'].items()} }")Auto with candidate control
You can limit which backbones are evaluated. Only the listed candidates are benchmarked — excluded backbones are never initialized or evaluated.
# Only evaluate two fast backbones
fast = Vyntri(
backbone='auto',
candidates=['mobilenet_v3_small', 'resnet18'],
seed=42,
)
fast.fit(train=s.train, val=s.val, progress=False)
sel = fast.metadata_['selection']
print(f"selected from fast set: {sel['selected']}")
print(f"candidates evaluated: {sel['candidates']}")