mirror of
https://www.modelscope.cn/PaddlePaddle/PaddleOCR-VL.git
synced 2026-07-16 13:42:55 +08:00
Update README.md (batch 1/1)
This commit is contained in:
systemd
@ -27,11 +27,10 @@ from transformers.activations import ACT2FN, GELUActivation
|
||||
from transformers.cache_utils import (
|
||||
Cache,
|
||||
DynamicCache,
|
||||
SlidingWindowCache,
|
||||
StaticCache,
|
||||
)
|
||||
from transformers.generation import GenerationMixin
|
||||
from transformers.integrations import use_kernel_forward_from_hub
|
||||
from transformers.masking_utils import create_causal_mask
|
||||
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
|
||||
from transformers.modeling_layers import GradientCheckpointingLayer
|
||||
from transformers.modeling_outputs import (
|
||||
@ -604,12 +603,13 @@ class Ernie4_5Model(Ernie4_5PreTrainedModel):
|
||||
elif position_ids.dim() == 2:
|
||||
position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
|
||||
|
||||
causal_mask = self._update_causal_mask(
|
||||
attention_mask,
|
||||
inputs_embeds,
|
||||
cache_position,
|
||||
past_key_values,
|
||||
output_attentions,
|
||||
causal_mask = create_causal_mask(
|
||||
config=self.config,
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
past_key_values=past_key_values,
|
||||
position_ids=position_ids,
|
||||
cache_position=cache_position,
|
||||
)
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
@ -632,170 +632,6 @@ class Ernie4_5Model(Ernie4_5PreTrainedModel):
|
||||
past_key_values=past_key_values,
|
||||
)
|
||||
|
||||
def _update_causal_mask(
|
||||
self,
|
||||
attention_mask: torch.Tensor,
|
||||
input_tensor: torch.Tensor,
|
||||
cache_position: torch.Tensor,
|
||||
past_key_values: Cache,
|
||||
output_attentions: bool = False,
|
||||
):
|
||||
if self.config._attn_implementation == "flash_attention_2":
|
||||
if attention_mask is not None and past_key_values is not None:
|
||||
is_padding_right = (
|
||||
attention_mask[:, -1].sum().item() != input_tensor.size()[0]
|
||||
)
|
||||
if is_padding_right:
|
||||
raise ValueError
|
||||
if attention_mask is not None and 0.0 in attention_mask:
|
||||
return attention_mask
|
||||
return None
|
||||
|
||||
# For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
|
||||
# order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
|
||||
# to infer the attention mask.
|
||||
past_seen_tokens = (
|
||||
past_key_values.get_seq_length() if past_key_values is not None else 0
|
||||
)
|
||||
using_static_cache = isinstance(past_key_values, StaticCache)
|
||||
using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache)
|
||||
|
||||
# When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
|
||||
if (
|
||||
self.config._attn_implementation == "sdpa"
|
||||
and not (using_static_cache or using_sliding_window_cache)
|
||||
and not output_attentions
|
||||
):
|
||||
if AttentionMaskConverter._ignore_causal_mask_sdpa(
|
||||
attention_mask,
|
||||
inputs_embeds=input_tensor,
|
||||
past_key_values_length=past_seen_tokens,
|
||||
sliding_window=self.config.sliding_window,
|
||||
is_training=self.training,
|
||||
):
|
||||
return None
|
||||
|
||||
dtype, device = input_tensor.dtype, input_tensor.device
|
||||
min_dtype = torch.finfo(dtype).min
|
||||
sequence_length = input_tensor.shape[1]
|
||||
# SlidingWindowCache or StaticCache
|
||||
if using_sliding_window_cache or using_static_cache:
|
||||
target_length = past_key_values.get_max_cache_shape()
|
||||
# DynamicCache or no cache
|
||||
else:
|
||||
target_length = (
|
||||
attention_mask.shape[-1]
|
||||
if isinstance(attention_mask, torch.Tensor)
|
||||
else past_seen_tokens + sequence_length + 1
|
||||
)
|
||||
|
||||
# In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
|
||||
causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
|
||||
attention_mask,
|
||||
sequence_length=sequence_length,
|
||||
target_length=target_length,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
cache_position=cache_position,
|
||||
batch_size=input_tensor.shape[0],
|
||||
config=self.config,
|
||||
past_key_values=past_key_values,
|
||||
)
|
||||
|
||||
if (
|
||||
self.config._attn_implementation == "sdpa"
|
||||
and attention_mask is not None
|
||||
and attention_mask.device.type in ["cuda", "xpu"]
|
||||
and not output_attentions
|
||||
):
|
||||
# Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
|
||||
# using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
|
||||
# Details: https://github.com/pytorch/pytorch/issues/110213
|
||||
causal_mask = AttentionMaskConverter._unmask_unattended(
|
||||
causal_mask, min_dtype
|
||||
)
|
||||
|
||||
return causal_mask
|
||||
|
||||
@staticmethod
|
||||
def _prepare_4d_causal_attention_mask_with_cache_position(
|
||||
attention_mask: torch.Tensor,
|
||||
sequence_length: int,
|
||||
target_length: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
cache_position: torch.Tensor,
|
||||
batch_size: int,
|
||||
config: PaddleOCRVLConfig,
|
||||
past_key_values: Cache,
|
||||
):
|
||||
"""
|
||||
Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
|
||||
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
|
||||
|
||||
Args:
|
||||
attention_mask (`torch.Tensor`):
|
||||
A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
|
||||
sequence_length (`int`):
|
||||
The sequence length being processed.
|
||||
target_length (`int`):
|
||||
The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
|
||||
dtype (`torch.dtype`):
|
||||
The dtype to use for the 4D attention mask.
|
||||
device (`torch.device`):
|
||||
The device to place the 4D attention mask on.
|
||||
cache_position (`torch.Tensor`):
|
||||
Indices depicting the position of the input sequence tokens in the sequence.
|
||||
batch_size (`torch.Tensor`):
|
||||
Batch size.
|
||||
config (`PaddleOCRVLConfig`):
|
||||
The model's configuration class
|
||||
past_key_values (`Cache`):
|
||||
The cache class that is being used currently to generate
|
||||
"""
|
||||
if attention_mask is not None and attention_mask.dim() == 4:
|
||||
# In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
|
||||
causal_mask = attention_mask
|
||||
else:
|
||||
min_dtype = torch.finfo(dtype).min
|
||||
causal_mask = torch.full(
|
||||
(sequence_length, target_length),
|
||||
fill_value=min_dtype,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
diagonal_attend_mask = torch.arange(
|
||||
target_length, device=device
|
||||
) > cache_position.reshape(-1, 1)
|
||||
if config.sliding_window is not None:
|
||||
# if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also
|
||||
# the check is needed to verify is current checkpoint was trained with sliding window or not
|
||||
if (
|
||||
not isinstance(past_key_values, SlidingWindowCache)
|
||||
or sequence_length > target_length
|
||||
):
|
||||
sliding_attend_mask = torch.arange(
|
||||
target_length, device=device
|
||||
) <= (cache_position.reshape(-1, 1) - config.sliding_window)
|
||||
diagonal_attend_mask.bitwise_or_(sliding_attend_mask)
|
||||
causal_mask *= diagonal_attend_mask
|
||||
causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
|
||||
if attention_mask is not None:
|
||||
causal_mask = (
|
||||
causal_mask.clone()
|
||||
) # copy to contiguous memory for in-place edit
|
||||
if attention_mask.shape[-1] > target_length:
|
||||
attention_mask = attention_mask[:, :target_length]
|
||||
mask_length = attention_mask.shape[-1]
|
||||
padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[
|
||||
:, None, None, :
|
||||
].to(causal_mask.device)
|
||||
padding_mask = padding_mask == 0
|
||||
causal_mask[:, :, :, :mask_length] = causal_mask[
|
||||
:, :, :, :mask_length
|
||||
].masked_fill(padding_mask, min_dtype)
|
||||
return causal_mask
|
||||
|
||||
|
||||
class Ernie4_5ForCausalLM(Ernie4_5PreTrainedModel, GenerationMixin):
|
||||
_tied_weights_keys = ["lm_head.weight"]
|
||||
@ -1033,7 +869,7 @@ class Projector(nn.Module):
|
||||
return hidden_states.view(*dims, -1)
|
||||
|
||||
|
||||
class SiglipVisionEmbeddings(nn.Module):
|
||||
class PaddleOCRVisionEmbeddings(nn.Module):
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@ -1217,7 +1053,7 @@ def eager_attention_forward(
|
||||
return attn_output, attn_weights
|
||||
|
||||
|
||||
class SiglipAttention(nn.Module):
|
||||
class PaddleOCRAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
@ -1348,8 +1184,8 @@ class SiglipAttention(nn.Module):
|
||||
return attn_output, attn_weights
|
||||
|
||||
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
|
||||
class SiglipMLP(nn.Module):
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->PaddleOCR
|
||||
class PaddleOCRMLP(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@ -1364,14 +1200,14 @@ class SiglipMLP(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class SiglipEncoderLayer(nn.Module):
|
||||
class PaddleOCREncoderLayer(nn.Module):
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
super().__init__()
|
||||
self.embed_dim = config.hidden_size
|
||||
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
self.self_attn = SiglipAttention(config)
|
||||
self.self_attn = PaddleOCRAttention(config)
|
||||
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
self.mlp = SiglipMLP(config)
|
||||
self.mlp = PaddleOCRMLP(config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@ -1416,23 +1252,23 @@ class SiglipEncoderLayer(nn.Module):
|
||||
return outputs
|
||||
|
||||
|
||||
class SiglipPreTrainedModel(PreTrainedModel):
|
||||
class PaddleOCRPreTrainedModel(PreTrainedModel):
|
||||
config_class = PaddleOCRVLConfig
|
||||
base_model_prefix = "siglip"
|
||||
base_model_prefix = "PaddleOCR"
|
||||
supports_gradient_checkpointing = True
|
||||
|
||||
_no_split_modules = [
|
||||
"SiglipTextEmbeddings",
|
||||
"SiglipEncoderLayer",
|
||||
"SiglipVisionEmbeddings",
|
||||
"SiglipMultiheadAttentionPoolingHead",
|
||||
"PaddleOCRTextEmbeddings",
|
||||
"PaddleOCREncoderLayer",
|
||||
"PaddleOCRVisionEmbeddings",
|
||||
"PaddleOCRMultiheadAttentionPoolingHead",
|
||||
]
|
||||
_supports_flash_attn_2 = True
|
||||
_supports_sdpa = True
|
||||
|
||||
def _init_weights(self, module):
|
||||
"""Initialize the weights"""
|
||||
if isinstance(module, SiglipVisionEmbeddings):
|
||||
if isinstance(module, PaddleOCRVisionEmbeddings):
|
||||
width = (
|
||||
self.config.vision_config.hidden_size
|
||||
if isinstance(self.config, PaddleOCRVLConfig)
|
||||
@ -1441,7 +1277,7 @@ class SiglipPreTrainedModel(PreTrainedModel):
|
||||
nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
|
||||
elif isinstance(module, nn.Embedding):
|
||||
default_flax_embed_init(module.weight)
|
||||
elif isinstance(module, SiglipAttention):
|
||||
elif isinstance(module, PaddleOCRAttention):
|
||||
nn.init.xavier_uniform_(module.q_proj.weight)
|
||||
nn.init.xavier_uniform_(module.k_proj.weight)
|
||||
nn.init.xavier_uniform_(module.v_proj.weight)
|
||||
@ -1450,12 +1286,12 @@ class SiglipPreTrainedModel(PreTrainedModel):
|
||||
nn.init.zeros_(module.k_proj.bias)
|
||||
nn.init.zeros_(module.v_proj.bias)
|
||||
nn.init.zeros_(module.out_proj.bias)
|
||||
elif isinstance(module, SiglipMLP):
|
||||
elif isinstance(module, PaddleOCRMLP):
|
||||
nn.init.xavier_uniform_(module.fc1.weight)
|
||||
nn.init.xavier_uniform_(module.fc2.weight)
|
||||
nn.init.normal_(module.fc1.bias, std=1e-6)
|
||||
nn.init.normal_(module.fc2.bias, std=1e-6)
|
||||
elif isinstance(module, SiglipMultiheadAttentionPoolingHead):
|
||||
elif isinstance(module, PaddleOCRMultiheadAttentionPoolingHead):
|
||||
nn.init.xavier_uniform_(module.probe.data)
|
||||
nn.init.xavier_uniform_(module.attention.in_proj_weight.data)
|
||||
nn.init.zeros_(module.attention.in_proj_bias.data)
|
||||
@ -1468,11 +1304,11 @@ class SiglipPreTrainedModel(PreTrainedModel):
|
||||
module.weight.data.fill_(1.0)
|
||||
|
||||
|
||||
# Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->Siglip
|
||||
class SiglipEncoder(nn.Module):
|
||||
# Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->PaddleOCR
|
||||
class PaddleOCREncoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
||||
[`SiglipEncoderLayer`].
|
||||
[`PaddleOCREncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: PaddleOCRVLConfig
|
||||
@ -1485,7 +1321,7 @@ class SiglipEncoder(nn.Module):
|
||||
num_heads = config.num_attention_heads
|
||||
head_dim = embed_dim // num_heads
|
||||
self.layers = nn.ModuleList(
|
||||
[SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)]
|
||||
[PaddleOCREncoderLayer(config) for _ in range(config.num_hidden_layers)]
|
||||
)
|
||||
self.rotary_pos_emb = SigLIPRotaryEmbedding(head_dim // 2)
|
||||
self.gradient_checkpointing = False
|
||||
@ -1703,20 +1539,20 @@ class SiglipEncoder(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class SiglipVisionTransformer(nn.Module):
|
||||
class PaddleOCRVisionTransformer(nn.Module):
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
embed_dim = config.hidden_size
|
||||
|
||||
self.embeddings = SiglipVisionEmbeddings(config)
|
||||
self.encoder = SiglipEncoder(config)
|
||||
self.embeddings = PaddleOCRVisionEmbeddings(config)
|
||||
self.encoder = PaddleOCREncoder(config)
|
||||
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
self.use_head = (
|
||||
True if not hasattr(config, "vision_use_head") else config.vision_use_head
|
||||
)
|
||||
if self.use_head:
|
||||
self.head = SiglipMultiheadAttentionPoolingHead(config)
|
||||
self.head = PaddleOCRMultiheadAttentionPoolingHead(config)
|
||||
|
||||
# @can_return_tuple
|
||||
def forward(
|
||||
@ -1861,7 +1697,7 @@ class SiglipVisionTransformer(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
||||
class PaddleOCRMultiheadAttentionPoolingHead(nn.Module):
|
||||
"""Multihead Attention Pooling."""
|
||||
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
@ -1872,7 +1708,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
||||
config.hidden_size, config.num_attention_heads, batch_first=True
|
||||
)
|
||||
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
||||
self.mlp = SiglipMLP(config)
|
||||
self.mlp = PaddleOCRMLP(config)
|
||||
|
||||
def forward(self, hidden_state, key_padding_mask=None):
|
||||
batch_size = hidden_state.shape[0]
|
||||
@ -1889,14 +1725,14 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
||||
return hidden_state[:, 0]
|
||||
|
||||
|
||||
class SiglipVisionModel(SiglipPreTrainedModel):
|
||||
class PaddleOCRVisionModel(PaddleOCRPreTrainedModel):
|
||||
config_class = PaddleOCRVisionConfig
|
||||
main_input_name = "pixel_values"
|
||||
|
||||
def __init__(self, config: PaddleOCRVisionConfig):
|
||||
super().__init__(config)
|
||||
|
||||
self.vision_model = SiglipVisionTransformer(config)
|
||||
self.vision_model = PaddleOCRVisionTransformer(config)
|
||||
|
||||
# Initialize weights and apply final processing
|
||||
self.post_init()
|
||||
@ -1922,29 +1758,6 @@ class SiglipVisionModel(SiglipPreTrainedModel):
|
||||
use_rope: Optional[bool] = False,
|
||||
window_size: Optional[bool] = -1,
|
||||
) -> BaseModelOutputWithPooling:
|
||||
r"""
|
||||
Returns:
|
||||
|
||||
Examples:
|
||||
|
||||
```python
|
||||
>>> from PIL import Image
|
||||
>>> import requests
|
||||
>>> from transformers import AutoProcessor, SiglipVisionModel
|
||||
|
||||
>>> model = SiglipVisionModel.from_pretrained("google/siglip-base-patch16-224")
|
||||
>>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
|
||||
|
||||
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
||||
>>> image = Image.open(requests.get(url, stream=True).raw)
|
||||
|
||||
>>> inputs = processor(images=image, return_tensors="pt")
|
||||
|
||||
>>> outputs = model(**inputs)
|
||||
>>> last_hidden_state = outputs.last_hidden_state
|
||||
>>> pooled_output = outputs.pooler_output # pooled features
|
||||
```"""
|
||||
|
||||
return self.vision_model(
|
||||
pixel_values=pixel_values,
|
||||
output_attentions=output_attentions,
|
||||
@ -2055,12 +1868,12 @@ class PaddleOCRVLCausalLMOutputWithPast(ModelOutput):
|
||||
class PaddleOCRVLForConditionalGeneration(Ernie4_5PreTrainedModel, GenerationMixin):
|
||||
_tied_weights_keys = ["lm_head.weight"]
|
||||
config_class = PaddleOCRVLConfig
|
||||
_no_split_modules = ["Ernie4_5_DecoderLayer", "SiglipEncoderLayer"]
|
||||
_no_split_modules = ["Ernie4_5_DecoderLayer", "PaddleOCREncoderLayer"]
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.mlp_AR = Projector(config, config.vision_config)
|
||||
self.visual = SiglipVisionModel(config.vision_config)
|
||||
self.visual = PaddleOCRVisionModel(config.vision_config)
|
||||
self.model = Ernie4_5Model(config)
|
||||
self.vocab_size = config.vocab_size
|
||||
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
||||
|
||||
Reference in New Issue
Block a user