diff --git a/README.md b/README.md
index 52b7b95..ba6c24e 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ PaddleOCR-VL: Boosting Multilingual Document Parsing via a 0.9B Ultra-Compact Vi
[](https://x.com/PaddlePaddle)
[](./LICENSE)
-**🔥 Official Demo**: [Baidu AI Studio](https://aistudio.baidu.com/application/detail/98365) |
+**🔥 Official Website**: [Baidu AI Studio](https://aistudio.baidu.com/paddleocr) |
**📝 arXiv**: [Technical Report](https://arxiv.org/pdf/2510.14528)
@@ -72,9 +72,11 @@ PaddleOCR-VL: Boosting Multilingual Document Parsing via a 0.9B Ultra-Compact Vi
## News
-* ```2025.10.16``` 🚀 We release [PaddleOCR-VL](https://github.com/PaddlePaddle/PaddleOCR), — a multilingual documents parsing via a 0.9B Ultra-Compact Vision-Language Model with SOTA performance.
-* ```2025.10.29``` Supports calling the core module PaddleOCR-VL-0.9B of PaddleOCR-VL via the `transformers` library.
+* ```2025.11.07``` 🚀 Enabled `flash-attn` in the `transformers` library to achieve faster inference with PaddleOCR-VL-0.9B.
+* ```2025.11.04``` 🌟 PaddleOCR-VL-0.9B is now officially supported on `vLLM` .
+* ```2025.10.29``` 🤗 Supports calling the core module PaddleOCR-VL-0.9B of PaddleOCR-VL via the `transformers` library.
+* ```2025.10.16``` 🚀 We release [PaddleOCR-VL](https://github.com/PaddlePaddle/PaddleOCR), — a multilingual documents parsing via a 0.9B Ultra-Compact Vision-Language Model with SOTA performance.
## Usage
@@ -83,27 +85,24 @@ PaddleOCR-VL: Boosting Multilingual Document Parsing via a 0.9B Ultra-Compact Vi
Install [PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick) and [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR):
```bash
-python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
-python -m pip install -U "paddleocr[doc-parser]"
-python -m pip install https://paddle-whl.bj.bcebos.com/nightly/cu126/safetensors/safetensors-0.6.2.dev0-cp38-abi3-linux_x86_64.whl
+# The following command installs the PaddlePaddle version for CUDA 12.6. For other CUDA versions and the CPU version, please refer to https://www.paddlepaddle.org.cn/en/install/quick?docurl=/documentation/docs/en/develop/install/pip/linux-pip_en.html
+python -m pip install paddlepaddle-gpu==3.2.1 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+python -m pip install -U "paddleocr[doc-parser]>=3.4.0"
```
-> For Windows users, please use WSL or a Docker container.
-
-
### Basic Usage
CLI usage:
```bash
-paddleocr doc_parser -i https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png
+paddleocr doc_parser -i https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png --pipeline_version v1
```
Python API usage:
```python
from paddleocr import PaddleOCRVL
-pipeline = PaddleOCRVL()
+pipeline = PaddleOCRVL(pipeline_version="v1")
output = pipeline.predict("https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png")
for res in output:
res.print()
@@ -113,26 +112,38 @@ for res in output:
### Accelerate VLM Inference via Optimized Inference Servers
-1. Start the VLM inference server (the default port is `8080`):
+1. Start the VLM inference server:
- ```bash
- docker run \
- --rm \
- --gpus all \
- --network host \
- ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddlex-genai-vllm-server
- ```
+ You can start the vLLM inference service using one of two methods:
+
+ - Method 1: PaddleOCR method
+
+ ```bash
+ docker run \
+ --rm \
+ --gpus all \
+ --network host \
+ ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleocr-genai-vllm-server:latest-nvidia-gpu \
+ paddleocr genai_server --model_name PaddleOCR-VL-0.9B --host 0.0.0.0 --port 8080 --backend vllm
+ ```
+
+ - Method 2: vLLM method
+
+ [vLLM: PaddleOCR-VL Usage Guide](https://docs.vllm.ai/projects/recipes/en/latest/PaddlePaddle/PaddleOCR-VL.html)
+
2. Call the PaddleOCR CLI or Python API:
```bash
paddleocr doc_parser \
-i https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png \
+ --pipeline_version v1 \
--vl_rec_backend vllm-server \
--vl_rec_server_url http://127.0.0.1:8080/v1
```
+
```python
from paddleocr import PaddleOCRVL
- pipeline = PaddleOCRVL(vl_rec_backend="vllm-server", vl_rec_server_url="http://127.0.0.1:8080/v1")
+ pipeline = PaddleOCRVL(pipeline_version="v1", vl_rec_backend="vllm-server", vl_rec_server_url="http://127.0.0.1:8080/v1")
output = pipeline.predict("https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png")
for res in output:
res.print()
@@ -154,9 +165,14 @@ from PIL import Image
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
+# ---- Settings ----
+model_path = "PaddlePaddle/PaddleOCR-VL"
+image_path = "test.png"
+task = "ocr" # Options: 'ocr' | 'table' | 'chart' | 'formula'
+# ------------------
+
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-CHOSEN_TASK = "ocr" # Options: 'ocr' | 'table' | 'chart' | 'formula'
PROMPTS = {
"ocr": "OCR:",
"table": "Table Recognition:",
@@ -164,8 +180,6 @@ PROMPTS = {
"chart": "Chart Recognition:",
}
-model_path = "PaddlePaddle/PaddleOCR-VL"
-image_path = "test.png"
image = Image.open(image_path).convert("RGB")
model = AutoModelForCausalLM.from_pretrained(
@@ -177,7 +191,7 @@ messages = [
{"role": "user",
"content": [
{"type": "image", "image": image},
- {"type": "text", "text": PROMPTS[CHOSEN_TASK]},
+ {"type": "text", "text": PROMPTS[task]},
]
}
]
@@ -186,7 +200,7 @@ inputs = processor.apply_chat_template(
tokenize=True,
add_generation_prompt=True,
return_dict=True,
- return_tensors="pt"
+ return_tensors="pt"
).to(DEVICE)
outputs = model.generate(**inputs, max_new_tokens=1024)
@@ -194,6 +208,73 @@ outputs = processor.batch_decode(outputs, skip_special_tokens=True)[0]
print(outputs)
```
+
+👉 Click to expand: Use flash-attn to boost performance and reduce memory usage
+
+```shell
+# ensure the flash-attn2 is installed
+pip install flash-attn --no-build-isolation
+```
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor
+from PIL import Image
+
+# ---- Settings ----
+model_path = "PaddlePaddle/PaddleOCR-VL"
+image_path = "test.png"
+task = "ocr" # ← change to "table" | "chart" | "formula"
+# ------------------
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+model = AutoModelForCausalLM.from_pretrained(
+ model_path,
+ trust_remote_code=True,
+ torch_dtype=torch.bfloat16,
+ attn_implementation="flash_attention_2",
+).to(dtype=torch.bfloat16, device=DEVICE).eval()
+processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+
+PROMPTS = {
+ "ocr": "OCR:",
+ "table": "Table Recognition:",
+ "chart": "Chart Recognition:",
+ "formula": "Formula Recognition:",
+}
+messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "image": Image.open(image_path).convert("RGB")},
+ {"type": "text", "text": PROMPTS[task]}
+ ]
+ }
+]
+
+inputs = processor.apply_chat_template(
+ messages,
+ tokenize=True,
+ add_generation_prompt=True,
+ return_dict=True,
+ return_tensors="pt"
+).to(DEVICE)
+
+with torch.inference_mode():
+ out = model.generate(
+ **inputs,
+ max_new_tokens=1024,
+ do_sample=False,
+ use_cache=True
+ )
+
+outputs = processor.batch_decode(out, skip_special_tokens=True)[0]
+print(outputs)
+```
+
+
+
## Performance
### Page-Level Document Parsing
@@ -346,4 +427,4 @@ If you find PaddleOCR-VL helpful, feel free to give us a star and citation.
primaryClass={cs.CV},
url={https://arxiv.org/abs/2510.14528},
}
-```
\ No newline at end of file
+```
diff --git a/chat_template.jinja b/chat_template.jinja
index 3583ca3..7b55077 100644
--- a/chat_template.jinja
+++ b/chat_template.jinja
@@ -7,16 +7,13 @@
{%- if not eos_token is defined -%}
{%- set eos_token = "" -%}
{%- endif -%}
-{%- if not image_token is defined -%}
- {%- set image_token = "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>" -%}
-{%- endif -%}
{{- cls_token -}}
{%- for message in messages -%}
{%- if message["role"] == "user" -%}
{{- "User: " -}}
{%- for content in message["content"] -%}
{%- if content["type"] == "image" -%}
- {{ image_token }}
+ {{ "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>" }}
{%- endif -%}
{%- endfor -%}
{%- for content in message["content"] -%}
diff --git a/config.json b/config.json
index 2617b7e..c547114 100644
--- a/config.json
+++ b/config.json
@@ -44,12 +44,12 @@
"video_token_id": 101307,
"vision_config": {
"architectures": [
- "SiglipVisionModel"
+ "PaddleOCRVisionModel"
],
"attention_dropout": 0.0,
"auto_map": {
"AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
- "AutoModel": "modeling_paddleocr_vl.SiglipVisionModel"
+ "AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
},
"hidden_act": "gelu_pytorch_tanh",
"hidden_size": 1152,
@@ -68,6 +68,7 @@
"torch_dtype": "bfloat16"
},
"vision_start_token_id": 101305,
+ "vision_end_token_id": 101306,
"vocab_size": 103424,
"weight_share_add_bias": true,
"use_3d_rope": true,
diff --git a/image_processing_paddleocr_vl.py b/image_processing_paddleocr_vl.py
new file mode 100644
index 0000000..f7e28fd
--- /dev/null
+++ b/image_processing_paddleocr_vl.py
@@ -0,0 +1,570 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Image processor class for PaddleOCR-VL."""
+
+import math
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import torch
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from torchvision.transforms import functional as TF
+from transformers.image_transforms import (
+ convert_to_rgb,
+ resize,
+ to_channel_dimension_format,
+)
+from transformers.image_utils import (
+ OPENAI_CLIP_MEAN,
+ OPENAI_CLIP_STD,
+ ChannelDimension,
+ PILImageResampling,
+ get_image_size,
+ infer_channel_dimension_format,
+ is_scaled_image,
+ is_valid_image,
+ make_list_of_images,
+ to_numpy_array,
+ valid_images,
+ validate_preprocess_arguments,
+)
+from transformers.utils import TensorType, is_vision_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_vision_available():
+ from PIL import Image
+
+ImageInput = Union[
+ "PIL.Image.Image",
+ np.ndarray,
+ "torch.Tensor",
+ List["PIL.Image.Image"],
+ List[np.ndarray],
+ List["torch.Tensor"],
+] # noqa
+
+
+VideoInput = Union[
+ List["PIL.Image.Image"],
+ "np.ndarray",
+ "torch.Tensor",
+ List["np.ndarray"],
+ List["torch.Tensor"],
+ List[List["PIL.Image.Image"]],
+ List[List["np.ndarrray"]],
+ List[List["torch.Tensor"]],
+] # noqa
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+ """
+ Accepts images in list or nested list format, and makes a list of images for preprocessing.
+
+ Args:
+ images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
+ The input image.
+
+ Returns:
+ list: A list of images.
+ """
+ if (
+ isinstance(images, (list, tuple))
+ and isinstance(images[0], (list, tuple))
+ and is_valid_image(images[0][0])
+ ):
+ return [img for img_list in images for img in img_list]
+
+ elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):
+ return images
+
+ elif is_valid_image(images):
+ return [images]
+
+ raise ValueError(f"Could not make batched images from {images}")
+
+
+def adjust_size(size, patch_size):
+ num_patches = size // patch_size
+ if num_patches % 2 != 0: # 如果是奇数，减1
+ num_patches -= 1
+ return num_patches * patch_size
+
+
+def make_batched_videos(videos) -> List[VideoInput]:
+ if (
+ isinstance(videos, (list, tuple))
+ and isinstance(videos[0], (list, tuple))
+ and is_valid_image(videos[0][0])
+ ):
+ return videos
+
+ elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]):
+ if isinstance(videos[0], Image.Image):
+ return [videos]
+ elif len(videos[0].shape) == 4:
+ return [list(video) for video in videos]
+
+ elif is_valid_image(videos) and len(videos.shape) == 4:
+ return [list(videos)]
+
+ raise ValueError(f"Could not make batched video from {videos}")
+
+
+def smart_resize(
+ height: int,
+ width: int,
+ factor: int = 28,
+ min_pixels: int = 28 * 28 * 130,
+ max_pixels: int = 28 * 28 * 1280,
+):
+ """Rescales the image so that the following conditions are met:
+
+ 1. Both dimensions (height and width) are divisible by 'factor'.
+
+ 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+ 3. The aspect ratio of the image is maintained as closely as possible.
+
+ """
+ # if height < factor or width < factor:
+ # raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
+ # if int(height < factor//4) + int(width < factor//4):
+ # raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor//4}")
+
+ if height < factor:
+ print(f"smart_resize: height={height} < factor={factor}, reset height=factor")
+ width = round((width * factor) / height)
+ height = factor
+
+ if width < factor:
+ print(f"smart_resize: width={width} < factor={factor}, reset width=factor")
+ height = round((height * factor) / width)
+ width = factor
+
+ if max(height, width) / min(height, width) > 200:
+ raise ValueError(
+ f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+ )
+ h_bar = round(height / factor) * factor
+ w_bar = round(width / factor) * factor
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = math.floor(height / beta / factor) * factor
+ w_bar = math.floor(width / beta / factor) * factor
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = math.ceil(height * beta / factor) * factor
+ w_bar = math.ceil(width * beta / factor) * factor
+ return h_bar, w_bar
+
+
+class PaddleOCRVLImageProcessor(BaseImageProcessor):
+ r"""
+ Constructs a Siglip image processor that dynamically resizes images based on the original images.
+
+ Args:
+ do_resize (`bool`, *optional*, defaults to `True`):
+ Whether to resize the image's (height, width) dimensions.
+ resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+ Resampling filter to use when resizing the image.
+ do_rescale (`bool`, *optional*, defaults to `True`):
+ Whether to rescale the image by the specified scale `rescale_factor`.
+ rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+ Scale factor to use if rescaling the image.
+ do_normalize (`bool`, *optional*, defaults to `True`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+ Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+ Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.
+ do_convert_rgb (`bool`, *optional*, defaults to `True`):
+ Whether to convert the image to RGB.
+ min_pixels (`int`, *optional*, defaults to `28 * 28 * 130`):
+ The min pixels of the image to resize the image.
+ max_pixels (`int`, *optional*, defaults to `28 * 28 * 1670`):
+ The max pixels of the image to resize the image.
+ patch_size (`int`, *optional*, defaults to 14):
+ The spacial patch size of the vision encoder.
+ temporal_patch_size (`int`, *optional*, defaults to 2):
+ The temporal patch size of the vision encoder.
+ merge_size (`int`, *optional*, defaults to 2):
+ The merge size of the vision encoder to llm encoder.
+ """
+
+ model_input_names = [
+ "pixel_values",
+ "image_grid_thw",
+ "pixel_values_videos",
+ "video_grid_thw",
+ ]
+
+ def __init__(
+ self,
+ do_resize: bool = True,
+ resample: PILImageResampling = PILImageResampling.BICUBIC,
+ do_rescale: bool = True,
+ rescale_factor: Union[int, float] = 1 / 255,
+ do_normalize: bool = True,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = True,
+ min_pixels: int = 28 * 28 * 130,
+ max_pixels: int = 28 * 28 * 1280,
+ patch_size: int = 14,
+ temporal_patch_size: int = 1,
+ merge_size: int = 2,
+ **kwargs,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.do_resize = do_resize
+ self.resample = resample
+ self.do_rescale = do_rescale
+ self.rescale_factor = rescale_factor
+ self.do_normalize = do_normalize
+ self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+ self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+ self.min_pixels = min_pixels
+ self.max_pixels = max_pixels
+ self.patch_size = patch_size
+ self.temporal_patch_size = temporal_patch_size
+ self.merge_size = merge_size
+ self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels} # not used
+ self.do_convert_rgb = do_convert_rgb
+
+ def mvit_rescale(self, image: Image.Image, merge_size: int = 2) -> Image.Image:
+ try:
+ w, h = image.size
+ except:
+ raise ValueError(str((type(image), image)))
+ patch_size = self.patch_size
+
+ if (w // patch_size) * (h // patch_size) > self.in_token_limit:
+ scale = math.sqrt(
+ self.in_token_limit / ((w // patch_size) * (h // patch_size))
+ )
+ new_w, new_h = int(w * scale), int(h * scale)
+
+ image = image.resize((new_w, new_h), Image.Resampling.BICUBIC)
+ if self.pad_input:
+ new_w, new_h = image.size
+ pad_size_h = merge_size * patch_size
+ pad_size_w = merge_size * patch_size
+
+ pad_h = (pad_size_h - new_h % pad_size_h) % pad_size_h
+ pad_w = (pad_size_w - new_w % pad_size_w) % pad_size_w
+
+ image = TF.pad(image, (0, 0, pad_w, pad_h))
+ else:
+ new_w, new_h = image.size
+ new_w = new_w - new_w % patch_size
+ new_h = new_h - new_h % patch_size
+
+ new_w = adjust_size(new_w, patch_size)
+ new_h = adjust_size(new_h, patch_size)
+
+ image = TF.center_crop(image, (new_h, new_w))
+
+ w, h = image.size
+ if w // patch_size >= 512 or h // patch_size >= 512:
+ new_h = min(patch_size * 510, h)
+ new_w = min(patch_size * 510, w)
+ image = TF.center_crop(image, (new_h, new_w))
+ # raise ValueError("Exceed pos emb")
+ return image
+
+ def _preprocess(
+ self,
+ images: Union[ImageInput, VideoInput],
+ do_resize: bool = None,
+ resample: PILImageResampling = None,
+ do_rescale: bool = None,
+ rescale_factor: float = None,
+ do_normalize: bool = None,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = None,
+ data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+ input_data_format: Optional[Union[str, ChannelDimension]] = None,
+ ):
+ """
+ Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+ Args:
+ images (`ImageInput`):
+ Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
+ vision_info (`List[Dict]`, *optional*):
+ Optional list of dictionaries containing additional information about vision inputs.
+ do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+ Whether to resize the image.
+ resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+ Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+ do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+ Whether to rescale the image.
+ rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+ Scale factor to use if rescaling the image.
+ do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+ Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+ Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+ Whether to convert the image to RGB.
+ data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+ The channel dimension format for the output image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - Unset: Use the channel dimension format of the input image.
+ input_data_format (`ChannelDimension` or `str`, *optional*):
+ The channel dimension format for the input image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+ """
+ images = make_list_of_images(images)
+
+ if input_data_format is None:
+ # We assume that all images have the same channel dimension format.
+ input_data_format = ChannelDimension.LAST if isinstance(images[0], Image.Image) else infer_channel_dimension_format(images[0])
+
+ if do_convert_rgb:
+ images = [convert_to_rgb(image) for image in images]
+
+ # All transformations expect numpy arrays.
+ images = [to_numpy_array(image) for image in images]
+
+ if is_scaled_image(images[0]) and do_rescale:
+ logger.warning_once(
+ "It looks like you are trying to rescale already rescaled images. If the input"
+ " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+ )
+
+ height, width = get_image_size(images[0], channel_dim=input_data_format)
+ resized_height, resized_width = height, width
+ processed_images = []
+
+ for image in images:
+ if do_resize:
+ resized_height, resized_width = smart_resize(
+ height,
+ width,
+ factor=self.patch_size * self.merge_size,
+ min_pixels=self.min_pixels,
+ max_pixels=self.max_pixels,
+ )
+ image = resize(
+ image,
+ size=(resized_height, resized_width),
+ resample=resample,
+ input_data_format=input_data_format,
+ )
+
+ if do_rescale:
+ image = self.rescale(
+ image, scale=rescale_factor, input_data_format=input_data_format
+ )
+
+ if do_normalize:
+ image = self.normalize(
+ image=image,
+ mean=image_mean,
+ std=image_std,
+ input_data_format=input_data_format,
+ )
+ image = to_channel_dimension_format(
+ image, data_format, input_channel_dim=input_data_format
+ )
+ processed_images.append(image)
+
+ patches = np.array(processed_images)
+ if data_format == ChannelDimension.LAST:
+ patches = patches.transpose(0, 3, 1, 2)
+ if patches.shape[0] == 1:
+ patches = np.tile(patches, (self.temporal_patch_size, 1, 1, 1))
+ init_patches = patches
+ channel = patches.shape[1]
+ grid_t = patches.shape[0] // self.temporal_patch_size
+ grid_h, grid_w = (
+ resized_height // self.patch_size,
+ resized_width // self.patch_size,
+ )
+ patches = patches.reshape(
+ grid_t,
+ self.temporal_patch_size,
+ channel,
+ grid_h,
+ self.patch_size,
+ grid_w,
+ self.patch_size,
+ )
+ patches = patches.transpose(0, 3, 5, 2, 1, 4, 6)
+ assert self.temporal_patch_size == 1
+ flatten_patches = patches.reshape(
+ grid_t * grid_h * grid_w, channel, self.patch_size, self.patch_size
+ )
+ return flatten_patches, (grid_t, grid_h, grid_w)
+
+ def preprocess(
+ self,
+ images: ImageInput,
+ videos: VideoInput = None,
+ do_resize: bool = None,
+ size: Dict[str, int] = None,
+ resample: PILImageResampling = None,
+ do_rescale: bool = None,
+ rescale_factor: float = None,
+ do_normalize: bool = None,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = None,
+ return_tensors: Optional[Union[str, TensorType]] = None,
+ data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+ input_data_format: Optional[Union[str, ChannelDimension]] = None,
+ ):
+ """
+ Args:
+ images (`ImageInput`):
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+ videos (`VideoInput`):
+ Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
+ passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
+ do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+ Whether to resize the image.
+ size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+ Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+ the longest edge resized to keep the input aspect ratio.
+ resample (`int`, *optional*, defaults to `self.resample`):
+ Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+ has an effect if `do_resize` is set to `True`.
+ do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+ Whether to rescale the image.
+ rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+ Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+ do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+ `True`.
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+ Whether to convert the image to RGB.
+ return_tensors (`str` or `TensorType`, *optional*):
+ The type of tensors to return. Can be one of:
+ - Unset: Return a list of `np.ndarray`.
+ - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+ - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+ - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+ - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+ data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+ The channel dimension format for the output image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - Unset: Use the channel dimension format of the input image.
+ input_data_format (`ChannelDimension` or `str`, *optional*):
+ The channel dimension format for the input image. If unset, the channel dimension format is inferred
+ from the input image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+ """
+ do_resize = do_resize if do_resize is not None else self.do_resize
+ size = size if size is not None else self.size
+ resample = resample if resample is not None else self.resample
+ do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+ rescale_factor = (
+ rescale_factor if rescale_factor is not None else self.rescale_factor
+ )
+ do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+ image_mean = image_mean if image_mean is not None else self.image_mean
+ image_std = image_std if image_std is not None else self.image_std
+ do_convert_rgb = (
+ do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+ )
+
+ if images is not None:
+ images = make_batched_images(images)
+ if videos is not None:
+ videos = make_batched_videos(videos)
+
+ if images is not None and not valid_images(images):
+ raise ValueError(
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+ "torch.Tensor, tf.Tensor or jax.ndarray."
+ )
+
+ validate_preprocess_arguments(
+ rescale_factor=rescale_factor,
+ do_normalize=do_normalize,
+ image_mean=image_mean,
+ image_std=image_std,
+ do_resize=do_resize,
+ size=size,
+ resample=resample,
+ )
+
+ if images is not None:
+ pixel_values, vision_grid_thws = [], []
+ for image in images:
+ patches, image_grid_thw = self._preprocess(
+ image,
+ do_resize=do_resize,
+ resample=resample,
+ do_rescale=do_rescale,
+ rescale_factor=rescale_factor,
+ do_normalize=do_normalize,
+ image_mean=image_mean,
+ image_std=image_std,
+ data_format=data_format,
+ do_convert_rgb=do_convert_rgb,
+ input_data_format=input_data_format,
+ )
+ pixel_values.extend(patches)
+ vision_grid_thws.append(image_grid_thw)
+ pixel_values = np.array(pixel_values)
+ vision_grid_thws = np.array(vision_grid_thws)
+ data = {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
+
+ if videos is not None:
+ pixel_values, vision_grid_thws = [], []
+ for images in videos:
+ patches, video_grid_thw = self._preprocess(
+ images,
+ do_resize=do_resize,
+ resample=resample,
+ do_rescale=do_rescale,
+ rescale_factor=rescale_factor,
+ do_normalize=do_normalize,
+ image_mean=image_mean,
+ image_std=image_std,
+ data_format=data_format,
+ do_convert_rgb=do_convert_rgb,
+ input_data_format=input_data_format,
+ )
+ pixel_values.extend(patches)
+ vision_grid_thws.append(video_grid_thw)
+ pixel_values = np.array(pixel_values)
+ vision_grid_thws = np.array(vision_grid_thws)
+ data = {
+ "pixel_values_videos": pixel_values,
+ "video_grid_thw": vision_grid_thws,
+ }
+
+ return BatchFeature(data=data, tensor_type=return_tensors)
diff --git a/modeling_paddleocr_vl.py b/modeling_paddleocr_vl.py
index 92fedc5..25dc764 100644
--- a/modeling_paddleocr_vl.py
+++ b/modeling_paddleocr_vl.py
@@ -27,11 +27,10 @@ from transformers.activations import ACT2FN, GELUActivation
from transformers.cache_utils import (
Cache,
DynamicCache,
- SlidingWindowCache,
- StaticCache,
)
from transformers.generation import GenerationMixin
from transformers.integrations import use_kernel_forward_from_hub
+from transformers.masking_utils import create_causal_mask
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
from transformers.modeling_layers import GradientCheckpointingLayer
from transformers.modeling_outputs import (
@@ -604,12 +603,13 @@ class Ernie4_5Model(Ernie4_5PreTrainedModel):
elif position_ids.dim() == 2:
position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
- causal_mask = self._update_causal_mask(
- attention_mask,
- inputs_embeds,
- cache_position,
- past_key_values,
- output_attentions,
+ causal_mask = create_causal_mask(
+ config=self.config,
+ inputs_embeds=inputs_embeds,
+ attention_mask=attention_mask,
+ past_key_values=past_key_values,
+ position_ids=position_ids,
+ cache_position=cache_position,
)
hidden_states = inputs_embeds
@@ -632,170 +632,6 @@ class Ernie4_5Model(Ernie4_5PreTrainedModel):
past_key_values=past_key_values,
)
- def _update_causal_mask(
- self,
- attention_mask: torch.Tensor,
- input_tensor: torch.Tensor,
- cache_position: torch.Tensor,
- past_key_values: Cache,
- output_attentions: bool = False,
- ):
- if self.config._attn_implementation == "flash_attention_2":
- if attention_mask is not None and past_key_values is not None:
- is_padding_right = (
- attention_mask[:, -1].sum().item() != input_tensor.size()[0]
- )
- if is_padding_right:
- raise ValueError
- if attention_mask is not None and 0.0 in attention_mask:
- return attention_mask
- return None
-
- # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
- # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
- # to infer the attention mask.
- past_seen_tokens = (
- past_key_values.get_seq_length() if past_key_values is not None else 0
- )
- using_static_cache = isinstance(past_key_values, StaticCache)
- using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache)
-
- # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
- if (
- self.config._attn_implementation == "sdpa"
- and not (using_static_cache or using_sliding_window_cache)
- and not output_attentions
- ):
- if AttentionMaskConverter._ignore_causal_mask_sdpa(
- attention_mask,
- inputs_embeds=input_tensor,
- past_key_values_length=past_seen_tokens,
- sliding_window=self.config.sliding_window,
- is_training=self.training,
- ):
- return None
-
- dtype, device = input_tensor.dtype, input_tensor.device
- min_dtype = torch.finfo(dtype).min
- sequence_length = input_tensor.shape[1]
- # SlidingWindowCache or StaticCache
- if using_sliding_window_cache or using_static_cache:
- target_length = past_key_values.get_max_cache_shape()
- # DynamicCache or no cache
- else:
- target_length = (
- attention_mask.shape[-1]
- if isinstance(attention_mask, torch.Tensor)
- else past_seen_tokens + sequence_length + 1
- )
-
- # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
- causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
- attention_mask,
- sequence_length=sequence_length,
- target_length=target_length,
- dtype=dtype,
- device=device,
- cache_position=cache_position,
- batch_size=input_tensor.shape[0],
- config=self.config,
- past_key_values=past_key_values,
- )
-
- if (
- self.config._attn_implementation == "sdpa"
- and attention_mask is not None
- and attention_mask.device.type in ["cuda", "xpu"]
- and not output_attentions
- ):
- # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
- # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
- # Details: https://github.com/pytorch/pytorch/issues/110213
- causal_mask = AttentionMaskConverter._unmask_unattended(
- causal_mask, min_dtype
- )
-
- return causal_mask
-
- @staticmethod
- def _prepare_4d_causal_attention_mask_with_cache_position(
- attention_mask: torch.Tensor,
- sequence_length: int,
- target_length: int,
- dtype: torch.dtype,
- device: torch.device,
- cache_position: torch.Tensor,
- batch_size: int,
- config: PaddleOCRVLConfig,
- past_key_values: Cache,
- ):
- """
- Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
- `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
-
- Args:
- attention_mask (`torch.Tensor`):
- A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
- sequence_length (`int`):
- The sequence length being processed.
- target_length (`int`):
- The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
- dtype (`torch.dtype`):
- The dtype to use for the 4D attention mask.
- device (`torch.device`):
- The device to place the 4D attention mask on.
- cache_position (`torch.Tensor`):
- Indices depicting the position of the input sequence tokens in the sequence.
- batch_size (`torch.Tensor`):
- Batch size.
- config (`PaddleOCRVLConfig`):
- The model's configuration class
- past_key_values (`Cache`):
- The cache class that is being used currently to generate
- """
- if attention_mask is not None and attention_mask.dim() == 4:
- # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
- causal_mask = attention_mask
- else:
- min_dtype = torch.finfo(dtype).min
- causal_mask = torch.full(
- (sequence_length, target_length),
- fill_value=min_dtype,
- dtype=dtype,
- device=device,
- )
- diagonal_attend_mask = torch.arange(
- target_length, device=device
- ) > cache_position.reshape(-1, 1)
- if config.sliding_window is not None:
- # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also
- # the check is needed to verify is current checkpoint was trained with sliding window or not
- if (
- not isinstance(past_key_values, SlidingWindowCache)
- or sequence_length > target_length
- ):
- sliding_attend_mask = torch.arange(
- target_length, device=device
- ) <= (cache_position.reshape(-1, 1) - config.sliding_window)
- diagonal_attend_mask.bitwise_or_(sliding_attend_mask)
- causal_mask *= diagonal_attend_mask
- causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
- if attention_mask is not None:
- causal_mask = (
- causal_mask.clone()
- ) # copy to contiguous memory for in-place edit
- if attention_mask.shape[-1] > target_length:
- attention_mask = attention_mask[:, :target_length]
- mask_length = attention_mask.shape[-1]
- padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[
- :, None, None, :
- ].to(causal_mask.device)
- padding_mask = padding_mask == 0
- causal_mask[:, :, :, :mask_length] = causal_mask[
- :, :, :, :mask_length
- ].masked_fill(padding_mask, min_dtype)
- return causal_mask
-
class Ernie4_5ForCausalLM(Ernie4_5PreTrainedModel, GenerationMixin):
_tied_weights_keys = ["lm_head.weight"]
@@ -1033,7 +869,7 @@ class Projector(nn.Module):
return hidden_states.view(*dims, -1)
-class SiglipVisionEmbeddings(nn.Module):
+class PaddleOCRVisionEmbeddings(nn.Module):
def __init__(self, config: PaddleOCRVisionConfig):
super().__init__()
self.config = config
@@ -1217,7 +1053,7 @@ def eager_attention_forward(
return attn_output, attn_weights
-class SiglipAttention(nn.Module):
+class PaddleOCRAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
def __init__(self, config: PaddleOCRVisionConfig):
@@ -1348,8 +1184,8 @@ class SiglipAttention(nn.Module):
return attn_output, attn_weights
-# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
-class SiglipMLP(nn.Module):
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->PaddleOCR
+class PaddleOCRMLP(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
@@ -1364,14 +1200,14 @@ class SiglipMLP(nn.Module):
return hidden_states
-class SiglipEncoderLayer(nn.Module):
+class PaddleOCREncoderLayer(nn.Module):
def __init__(self, config: PaddleOCRVisionConfig):
super().__init__()
self.embed_dim = config.hidden_size
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
- self.self_attn = SiglipAttention(config)
+ self.self_attn = PaddleOCRAttention(config)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
- self.mlp = SiglipMLP(config)
+ self.mlp = PaddleOCRMLP(config)
def forward(
self,
@@ -1416,23 +1252,23 @@ class SiglipEncoderLayer(nn.Module):
return outputs
-class SiglipPreTrainedModel(PreTrainedModel):
+class PaddleOCRPreTrainedModel(PreTrainedModel):
config_class = PaddleOCRVLConfig
- base_model_prefix = "siglip"
+ base_model_prefix = "PaddleOCR"
supports_gradient_checkpointing = True
_no_split_modules = [
- "SiglipTextEmbeddings",
- "SiglipEncoderLayer",
- "SiglipVisionEmbeddings",
- "SiglipMultiheadAttentionPoolingHead",
+ "PaddleOCRTextEmbeddings",
+ "PaddleOCREncoderLayer",
+ "PaddleOCRVisionEmbeddings",
+ "PaddleOCRMultiheadAttentionPoolingHead",
]
_supports_flash_attn_2 = True
_supports_sdpa = True
def _init_weights(self, module):
"""Initialize the weights"""
- if isinstance(module, SiglipVisionEmbeddings):
+ if isinstance(module, PaddleOCRVisionEmbeddings):
width = (
self.config.vision_config.hidden_size
if isinstance(self.config, PaddleOCRVLConfig)
@@ -1441,7 +1277,7 @@ class SiglipPreTrainedModel(PreTrainedModel):
nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
elif isinstance(module, nn.Embedding):
default_flax_embed_init(module.weight)
- elif isinstance(module, SiglipAttention):
+ elif isinstance(module, PaddleOCRAttention):
nn.init.xavier_uniform_(module.q_proj.weight)
nn.init.xavier_uniform_(module.k_proj.weight)
nn.init.xavier_uniform_(module.v_proj.weight)
@@ -1450,12 +1286,12 @@ class SiglipPreTrainedModel(PreTrainedModel):
nn.init.zeros_(module.k_proj.bias)
nn.init.zeros_(module.v_proj.bias)
nn.init.zeros_(module.out_proj.bias)
- elif isinstance(module, SiglipMLP):
+ elif isinstance(module, PaddleOCRMLP):
nn.init.xavier_uniform_(module.fc1.weight)
nn.init.xavier_uniform_(module.fc2.weight)
nn.init.normal_(module.fc1.bias, std=1e-6)
nn.init.normal_(module.fc2.bias, std=1e-6)
- elif isinstance(module, SiglipMultiheadAttentionPoolingHead):
+ elif isinstance(module, PaddleOCRMultiheadAttentionPoolingHead):
nn.init.xavier_uniform_(module.probe.data)
nn.init.xavier_uniform_(module.attention.in_proj_weight.data)
nn.init.zeros_(module.attention.in_proj_bias.data)
@@ -1468,11 +1304,11 @@ class SiglipPreTrainedModel(PreTrainedModel):
module.weight.data.fill_(1.0)
-# Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->Siglip
-class SiglipEncoder(nn.Module):
+# Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->PaddleOCR
+class PaddleOCREncoder(nn.Module):
"""
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
- [`SiglipEncoderLayer`].
+ [`PaddleOCREncoderLayer`].
Args:
config: PaddleOCRVLConfig
@@ -1485,7 +1321,7 @@ class SiglipEncoder(nn.Module):
num_heads = config.num_attention_heads
head_dim = embed_dim // num_heads
self.layers = nn.ModuleList(
- [SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)]
+ [PaddleOCREncoderLayer(config) for _ in range(config.num_hidden_layers)]
)
self.rotary_pos_emb = SigLIPRotaryEmbedding(head_dim // 2)
self.gradient_checkpointing = False
@@ -1703,20 +1539,20 @@ class SiglipEncoder(nn.Module):
)
-class SiglipVisionTransformer(nn.Module):
+class PaddleOCRVisionTransformer(nn.Module):
def __init__(self, config: PaddleOCRVisionConfig):
super().__init__()
self.config = config
embed_dim = config.hidden_size
- self.embeddings = SiglipVisionEmbeddings(config)
- self.encoder = SiglipEncoder(config)
+ self.embeddings = PaddleOCRVisionEmbeddings(config)
+ self.encoder = PaddleOCREncoder(config)
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.use_head = (
True if not hasattr(config, "vision_use_head") else config.vision_use_head
)
if self.use_head:
- self.head = SiglipMultiheadAttentionPoolingHead(config)
+ self.head = PaddleOCRMultiheadAttentionPoolingHead(config)
# @can_return_tuple
def forward(
@@ -1861,7 +1697,7 @@ class SiglipVisionTransformer(nn.Module):
)
-class SiglipMultiheadAttentionPoolingHead(nn.Module):
+class PaddleOCRMultiheadAttentionPoolingHead(nn.Module):
"""Multihead Attention Pooling."""
def __init__(self, config: PaddleOCRVisionConfig):
@@ -1872,7 +1708,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
config.hidden_size, config.num_attention_heads, batch_first=True
)
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
- self.mlp = SiglipMLP(config)
+ self.mlp = PaddleOCRMLP(config)
def forward(self, hidden_state, key_padding_mask=None):
batch_size = hidden_state.shape[0]
@@ -1889,14 +1725,14 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
return hidden_state[:, 0]
-class SiglipVisionModel(SiglipPreTrainedModel):
+class PaddleOCRVisionModel(PaddleOCRPreTrainedModel):
config_class = PaddleOCRVisionConfig
main_input_name = "pixel_values"
def __init__(self, config: PaddleOCRVisionConfig):
super().__init__(config)
- self.vision_model = SiglipVisionTransformer(config)
+ self.vision_model = PaddleOCRVisionTransformer(config)
# Initialize weights and apply final processing
self.post_init()
@@ -1922,29 +1758,6 @@ class SiglipVisionModel(SiglipPreTrainedModel):
use_rope: Optional[bool] = False,
window_size: Optional[bool] = -1,
) -> BaseModelOutputWithPooling:
- r"""
- Returns:
-
- Examples:
-
- ```python
- >>> from PIL import Image
- >>> import requests
- >>> from transformers import AutoProcessor, SiglipVisionModel
-
- >>> model = SiglipVisionModel.from_pretrained("google/siglip-base-patch16-224")
- >>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
-
- >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
- >>> image = Image.open(requests.get(url, stream=True).raw)
-
- >>> inputs = processor(images=image, return_tensors="pt")
-
- >>> outputs = model(**inputs)
- >>> last_hidden_state = outputs.last_hidden_state
- >>> pooled_output = outputs.pooler_output # pooled features
- ```"""
-
return self.vision_model(
pixel_values=pixel_values,
output_attentions=output_attentions,
@@ -2055,12 +1868,12 @@ class PaddleOCRVLCausalLMOutputWithPast(ModelOutput):
class PaddleOCRVLForConditionalGeneration(Ernie4_5PreTrainedModel, GenerationMixin):
_tied_weights_keys = ["lm_head.weight"]
config_class = PaddleOCRVLConfig
- _no_split_modules = ["Ernie4_5_DecoderLayer", "SiglipEncoderLayer"]
+ _no_split_modules = ["Ernie4_5_DecoderLayer", "PaddleOCREncoderLayer"]
def __init__(self, config):
super().__init__(config)
self.mlp_AR = Projector(config, config.vision_config)
- self.visual = SiglipVisionModel(config.vision_config)
+ self.visual = PaddleOCRVisionModel(config.vision_config)
self.model = Ernie4_5Model(config)
self.vocab_size = config.vocab_size
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
diff --git a/preprocessor_config.json b/preprocessor_config.json
index 1f0535a..039dc0b 100644
--- a/preprocessor_config.json
+++ b/preprocessor_config.json
@@ -1,6 +1,6 @@
{
"auto_map": {
- "AutoImageProcessor": "image_processing.SiglipImageProcessor",
+ "AutoImageProcessor": "image_processing_paddleocr_vl.PaddleOCRVLImageProcessor",
"AutoProcessor": "processing_paddleocr_vl.PaddleOCRVLProcessor"
},
"do_convert_rgb": true,
@@ -12,7 +12,7 @@
0.5,
0.5
],
- "image_processor_type": "SiglipImageProcessor",
+ "image_processor_type": "PaddleOCRVLImageProcessor",
"image_std": [
0.5,
0.5,
@@ -25,9 +25,5 @@
"processor_class": "PaddleOCRVLProcessor",
"resample": 3,
"rescale_factor": 0.00392156862745098,
- "size": {
- "max_pixels": 2822400,
- "min_pixels": 147384
- },
"temporal_patch_size": 1
}
diff --git a/tokenizer_config.json b/tokenizer_config.json
index 8ac0275..c8a4f4b 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -8324,18 +8324,19 @@
"<|video_pad|>"
],
"auto_map": {
- "AutoProcessor": "processing_ppocrvl.PPOCRVLProcessor"
+ "AutoProcessor": "processing_paddleocr_vl.PaddleOCRVLProcessor"
},
"bos_token": "",
"clean_up_tokenization_spaces": false,
"cls_token": "<|begin_of_sentence|>",
"eos_token": "",
+ "image_token": "<|IMAGE_PLACEHOLDER|>",
"extra_special_tokens": {},
"legacy": true,
"mask_token": "",
"model_max_length": 131072,
"pad_token": "",
- "processor_class": "PPOCRVLProcessor",
+ "processor_class": "PaddleOCRVLProcessor",
"sep_token": "<|end_of_sentence|>",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,