microsoft

/

Florence-2-large-ft

like 317

Follow

Microsoft 7.86k

Issue

There is a device placement issue when setting the device to mps.

Replication:

Run the sample ipynb script, but set device to mps instead of cuda (and move the inputs to mps as well:

from tqdm.auto import tqdm
model_id = 'microsoft/Florence-2-base-ft'

device = torch.device("cuda" if torch.cuda.is_available() else ("mps:0" if torch.backends.mps.is_available() else "cpu"))

print(f"Device set to {device}")

    with tqdm(total=1, desc=f"Loading Model to {device}") as pbar:
        model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True, torch_dtype='auto').eval()
        model.to(device)
        pbar.update(1)

processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)

task_prompt = '<DETAILED_CAPTION>'
run_example(task_prompt)

Error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[21], line 4
      2 start_time = time.time()
      3 task_prompt = '<CAPTION>'
----> 4 results = run_example(task_prompt)
      5 end_time = time.time()
      6 elapsed_time = end_time - start_time

Cell In[17], line 17, in run_example(task_prompt, text_input, device)
     11 inputs = {
     12     "input_ids": processor_output["input_ids"].to(device),
     13     "pixel_values": processor_output["pixel_values"].to(device, dtype=torch.float16)
     14 }
     16 # Generate with inputs already on correct device
---> 17 generated_ids = model.generate(
     18     input_ids=inputs["input_ids"],
     19     pixel_values=inputs["pixel_values"],
     20     max_new_tokens=1024,
     21     early_stopping=False,
     22     do_sample=False,
     23     num_beams=3,
     24 )
     26 generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
     27 parsed_answer = processor.post_process_generation(
     28     generated_text, 
     29     task=task_prompt, 
     30     image_size=(image.width, image.height)
     31 )

File ~/.cache/huggingface/modules/transformers_modules/microsoft/Florence-2-base-ft/9803f52844ec1ae5df004e6089262e9a23e527fd/modeling_florence2.py:2790, in Florence2ForConditionalGeneration.generate(self, input_ids, inputs_embeds, pixel_values, **kwargs)
   2787 if inputs_embeds is None:
   2788     # 1. Extra the input embeddings
   2789     if input_ids is not None:
-> 2790         inputs_embeds = self.get_input_embeddings()(input_ids)
   2791     # 2. Merge text and images
   2792     if pixel_values is not None:

File ~/TR/ADVANCED-vision/fine-tune/florence2/florEnv/lib/python3.12/site-packages/torch/nn/modules/module.py:1736, in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File ~/TR/ADVANCED-vision/fine-tune/florence2/florEnv/lib/python3.12/site-packages/torch/nn/modules/module.py:1747, in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File ~/TR/ADVANCED-vision/fine-tune/florence2/florEnv/lib/python3.12/site-packages/torch/nn/modules/sparse.py:190, in Embedding.forward(self, input)
    189 def forward(self, input: Tensor) -> Tensor:
--> 190     return F.embedding(
    191         input,
    192         self.weight,
    193         self.padding_idx,
    194         self.max_norm,
    195         self.norm_type,
    196         self.scale_grad_by_freq,
    197         self.sparse,
    198     )

File ~/TR/ADVANCED-vision/fine-tune/florence2/florEnv/lib/python3.12/site-packages/torch/nn/functional.py:2551, in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2545     # Note [embedding_renorm set_grad_enabled]
   2546     # XXX: equivalent to
   2547     # with torch.no_grad():
   2548     #   torch.embedding_renorm_
   2549     # remove once script supports set_grad_enabled
   2550     _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2551 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

RuntimeError: Placeholder storage has not been allocated on MPS device!