""" Self-contained processor shim for trust_remote_code. This processor: - loads frames from a video path (OpenCV) - applies InternVL2.5 `...*N...` token template - returns `vision_input`, `input_ids`, `attention_mask`, and `question_input_ids` compatible with `QTSplusInternLM2_ForCausalLM.generate`. """ from __future__ import annotations from dataclasses import dataclass from typing import List, Optional, Union import numpy as np from PIL import Image from transformers.feature_extraction_utils import BatchFeature from transformers.processing_utils import ProcessorMixin from transformers.tokenization_utils_base import PreTokenizedInput, TextInput def _uniform_indices(num_frames: int, vlen: int) -> List[int]: num_frames = max(int(num_frames), 1) if vlen <= 0: return [] if num_frames == 1: return [max(0, (vlen - 1) // 2)] last = vlen - 1 return [int(round(i * last / (num_frames - 1))) for i in range(num_frames)] def _load_video_frames_cv2(path: str, num_frames: int = 8) -> List[Image.Image]: import cv2 cap = cv2.VideoCapture(path) if not cap.isOpened(): raise FileNotFoundError(f"Failed to open video: {path}") vlen = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) if vlen <= 0: # Fallback: decode sequentially and take the first `num_frames`. frames: List[Image.Image] = [] while len(frames) < num_frames: ok, frame = cap.read() if not ok: break frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frames.append(Image.fromarray(frame)) cap.release() return frames indices = _uniform_indices(num_frames, vlen) frames = [] for idx in indices: cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx)) ok, frame = cap.read() if not ok: continue frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frames.append(Image.fromarray(frame)) cap.release() return frames @dataclass class QTSplusInternVL2_5_ProcessorKwargs: num_frames: int = 8 system_prompt: str = "You are a helpful assistant." class QTSplusInternVL2_5_Processor(ProcessorMixin): attributes = ["image_processor", "tokenizer"] image_processor_class = "AutoImageProcessor" tokenizer_class = "AutoTokenizer" def __init__(self, image_processor=None, tokenizer=None, **kwargs): super().__init__(image_processor=image_processor, tokenizer=tokenizer) self.img_start_token = "" self.img_end_token = "" self.img_context_token = "" # InternVL2.5 default: (448/14)^2 * (0.5^2) = 256 tokens per image. self.num_image_token = int(kwargs.pop("num_image_token", 256)) self.system_prompt = str(kwargs.pop("system_prompt", "You are a helpful assistant.")) def _build_image_tokens(self, num_images: int) -> str: num_images = max(int(num_images), 1) one = self.img_start_token + (self.img_context_token * int(self.num_image_token)) + self.img_end_token return one * num_images def __call__( self, text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, images=None, videos: Optional[Union[str, List[Image.Image]]] = None, return_tensors: Optional[str] = "pt", num_frames: int = 8, system_prompt: Optional[str] = None, **kwargs, ) -> BatchFeature: if text is None: raise ValueError("`text` is required") if isinstance(text, list): if len(text) != 1: raise ValueError("Only single-example processing is supported for now") text = text[0] if videos is not None and images is not None: raise ValueError("Pass only one of `videos` or `images`") if videos is not None: if isinstance(videos, str): frames = _load_video_frames_cv2(videos, num_frames=num_frames) elif isinstance(videos, list): frames = videos else: raise ValueError(f"Unsupported `videos` type: {type(videos)}") images = frames if images is None: raise ValueError("Either `videos` or `images` must be provided") if isinstance(images, Image.Image): images = [images] if not isinstance(images, list) or not images: raise ValueError("No frames/images loaded") img_tokens = self._build_image_tokens(num_images=len(images)) user_content = f"{img_tokens}\n{text}" messages = [ {"role": "system", "content": system_prompt or self.system_prompt}, {"role": "user", "content": user_content}, ] if not hasattr(self.tokenizer, "apply_chat_template"): raise ValueError("Tokenizer does not support apply_chat_template; missing chat_template.jinja?") prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) text_inputs = self.tokenizer( prompt, add_special_tokens=False, return_tensors=return_tensors, ) question_inputs = self.tokenizer( str(text), add_special_tokens=False, return_tensors=return_tensors, ) vision_inputs = self.image_processor(images=images, return_tensors=return_tensors) pixel_values = vision_inputs["pixel_values"] return BatchFeature( data={ "input_ids": text_inputs["input_ids"], "attention_mask": text_inputs.get("attention_mask"), "question_input_ids": question_inputs["input_ids"], "vision_input": pixel_values, }, tensor_type=return_tensors, )