immunopath-medgemma-v2

import torch
from transformers import AutoProcessor, AutoModelForImageTextToText, BitsAndBytesConfig
from peft import PeftModel
from PIL import Image

MODEL_ID = "google/medgemma-1.5-4b-it"
ADAPTER_REPO = "hetanshwaghela/immunopath-medgemma-v2"

# Load base model (4-bit quantized)
processor = AutoProcessor.from_pretrained(MODEL_ID)
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)
base_model = AutoModelForImageTextToText.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    quantization_config=bnb_config,
)

# Load LoRA adapter
model = PeftModel.from_pretrained(base_model, ADAPTER_REPO)
model.eval()

# Run inference on a histopathology patch
image = Image.open("patch_512x512.png").convert("RGB")

prompt = """Analyze this H&E histopathology patch from a lung cancer biopsy.
Predict the tumor immune microenvironment and return a JSON object with these fields:
- cd274_expression: "high" or "low"
- msi_status: "MSI-H" or "MSS"
- tme_subtype: one of "IE", "IE/F", "F", "D"
- til_fraction: float 0-1
- til_density: "high", "moderate", or "low"
- immune_phenotype: "inflamed", "excluded", or "desert"
- cd8_infiltration: "high", "moderate", or "low"
- immune_score: float 0-1"""

messages = [
    {"role": "user", "content": [
        {"type": "image", "image": image},
        {"type": "text", "text": prompt},
    ]}
]

inputs = processor.apply_chat_template(
    messages, add_generation_prompt=True, tokenize=True,
    return_dict=True, return_tensors="pt"
).to(model.device)

with torch.no_grad():
    output = model.generate(**inputs, max_new_tokens=512, do_sample=False)

response = processor.decode(output[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print(response)  # JSON with 8 biomarker predictions

@software{immunopath2026,
  title={ImmunoPath: H&E Histopathology to Immunotherapy Decision Support via Fine-Tuned MedGemma},
  author={Hetansh Waghela},
  year={2026},
  url={https://github.com/hetanshwaghela/immunopath}
}