vllm.transformers_utils.config ¶

def _iter_rope_parameters(config: PretrainedConfig) -> Iterator[dict[str, Any]]:
    """Yield a config's rope parameters, one dict per layer type if nested."""
    rope_parameters = getattr(config, "rope_parameters", None)
    if not isinstance(rope_parameters, dict):
        return

    if is_rope_parameters_nested(rope_parameters):
        yield from (p for p in rope_parameters.values() if isinstance(p, dict))
    else:
        yield rope_parameters

def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
    return config

def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """Update kwargs for AutoConfig initialization based on model_type."""
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs

def _mrope_section(config: PretrainedConfig) -> Sequence[int] | None:
    """Return the M-RoPE section this config declares, if any.

    `xdrope_section` is the legacy name HunYuan-VL checkpoints use for the same
    field; upstream Transformers normalises it to `mrope_section`.
    """
    from vllm.config.utils import getattr_iter

    names = ("mrope_section", "xdrope_section")

    for params in _iter_rope_parameters(config):
        for i, name in enumerate(names):
            section = params.get(name)
            if isinstance(section, (list, tuple)):
                if i > 0:
                    logger.warning_once(
                        "rope_parameters contains a deprecated key '%s'. "
                        "Please use the preferred key '%s' instead.",
                        name,
                        names[0],
                    )
                return section

    section = getattr_iter(config, names, None, warn=True)
    return section if isinstance(section, (list, tuple)) else None

def _patch_hf_transformers_allowed_layer_types(
    extra_layer_types: tuple[str, ...],
) -> None:
    """Extend transformers' ``ALLOWED_LAYER_TYPES`` so its strict config
    validation accepts layer types (e.g. ``deepseek_sparse_attention``) that a
    checkpoint declares but upstream transformers has not registered yet.
    """
    import transformers.configuration_utils as hf_configuration_utils

    missing = tuple(
        layer_type
        for layer_type in extra_layer_types
        if layer_type not in hf_configuration_utils.ALLOWED_LAYER_TYPES
    )
    if missing:
        hf_configuration_utils.ALLOWED_LAYER_TYPES += missing

def _patch_hf_transformers_nested_rope_validation() -> None:
    """Drop shared entries sitting alongside a nested ``rope_parameters`` dict.

    Transformers validates a dict with any layer-type key by iterating all of
    its values, so a shared entry next to the per-layer dicts raises an
    ``AttributeError``. The per-layer dicts already carry their own defaults by
    the time validation runs, so the shared entries can be dropped.
    """
    if hasattr(PretrainedConfig.validate_rope, "__vllm_nested_rope_patched__"):
        return

    _original_validate_rope = PretrainedConfig.validate_rope

    @wraps(_original_validate_rope)
    def patched_validate_rope(self, *args, **kwargs):
        rope_parameters = getattr(self, "rope_parameters", None)
        if isinstance(rope_parameters, dict):
            layer_types = set(rope_parameters) & set(
                hf_configuration_utils.ALLOWED_LAYER_TYPES
            )
            if shared_keys := set(rope_parameters) - layer_types:
                for key in shared_keys:
                    del rope_parameters[key]
                logger.warning(
                    "Dropping %s from rope_parameters; entries shared across "
                    "layer types are not supported",
                    sorted(shared_keys),
                )
        return _original_validate_rope(self, *args, **kwargs)

    patched_validate_rope.__vllm_nested_rope_patched__ = True  # type: ignore[attr-defined]
    PretrainedConfig.validate_rope = patched_validate_rope

def _patch_hf_transformers_validate_rope():
    """Transformers v5 moved the ignore_keys option from the method signature of
    validate_rope and replaced it with the ignore_keys_at_rope_validation parameter
    in the PreTrainedConfig class. This is a patch to make older versions of
    validate_rope() with the ignore_keys parameter work with newer versions of
    hf transformers (from v5 onwards)
    """
    if hasattr(PretrainedConfig.validate_rope, "__vllm_patched__"):
        return

    _original_validate_rope = PretrainedConfig.validate_rope

    @wraps(_original_validate_rope)
    def patched_validate_rope(self, *args, **kwargs):
        ignore_keys_param = kwargs.pop("ignore_keys", None)
        original_ignore_keys = self.ignore_keys_at_rope_validation
        if ignore_keys_param is not None:
            logger.warning(
                "validate_rope() was called with the legacy 'ignore_keys' "
                "argument; use 'ignore_keys_at_rope_validation' instead"
            )
        self.ignore_keys_at_rope_validation = original_ignore_keys or ignore_keys_param
        result = _original_validate_rope(self, *args, **kwargs)
        return result

    patched_validate_rope.__vllm_patched__ = True  # type: ignore[attr-defined]
    PretrainedConfig.validate_rope = patched_validate_rope

@cache
def checkpoint_has_lm_head(model: str, *, revision: str | None = None) -> bool | None:
    """Whether the checkpoint contains an `lm_head` tensor of its own.

    Args:
        model: Name or path of the model repository.
        revision: The specific model version to use.

    Returns:
        `None` if the checkpoint contents could not be determined, for example
        because it is not stored as safetensors.

    """
    metadata = get_safetensors_params_metadata(model, revision=revision)
    if not metadata:
        return None
    return any(name.endswith("lm_head.weight") for name in metadata)

def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()

def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
    No op for pure text models.
    """
    text_config = config.get_text_config()

    if text_config is not config and not hasattr(text_config, "num_attention_heads"):
        raise ValueError(
            "The text_config extracted from the model config does not have "
            "`num_attention_heads` attribute. This indicates a mismatch "
            "between the model config and vLLM's expectations. Please "
            "ensure that the model config is compatible with vLLM."
        )

    return text_config

@cache
def get_pooling_config(
    model: str,
    revision: str | None = "main",
) -> dict[str, Any] | None:
    """This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.

    Args:
        model: The name of the Hugging Face model.
        revision: The specific version of the model to use.
            Defaults to 'main'.

    Returns:
        A dictionary containing the pooling type and whether
            normalization is used, or None if no pooling configuration is found.

    """
    modules_file_name = "modules.json"

    modules_dict = None
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)

    if modules_dict is None:
        return None

    logger.info("Found sentence-transformers modules configuration.")

    pooling = next(
        (item for item in modules_dict if item["type"] in _ST_POOLING_MODULE_TYPES),
        None,
    )
    normalize = bool(
        next(
            (
                item
                for item in modules_dict
                if item["type"] in _ST_NORMALIZE_MODULE_TYPES
            ),
            False,
        )
    )

    if pooling:
        from vllm.config.pooler import SEQ_POOLING_TYPES, TOK_POOLING_TYPES

        pooling_file_name = "{}/config.json".format(pooling["path"])
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision) or {}

        logger.info("Found pooling configuration.")

        config: dict[str, Any] = {"use_activation": normalize}
        for key, val in pooling_dict.items():
            if key == "pooling_mode" and isinstance(val, str):
                pooling_name = val
            elif val is True:
                pooling_name = key
            else:
                continue

            pooling_type = parse_pooling_type(pooling_name)
            if pooling_type in SEQ_POOLING_TYPES:
                config["seq_pooling_type"] = pooling_type
            elif pooling_type in TOK_POOLING_TYPES:
                config["tok_pooling_type"] = pooling_type
            else:
                logger.debug("Skipping unrelated field: %r=%r", key, val)

        if not {"seq_pooling_type", "tok_pooling_type"} & config.keys():
            logger.warning(
                "Unable to determine Sentence Transformers pooling type from %s; "
                "unless configured explicitly, vLLM will fall back to the model "
                "architecture default.",
                pooling_file_name,
            )

        return config

    return None

def get_safetensors_params_metadata(
    model: str,
    *,
    revision: str | None = None,
) -> dict[str, Any]:
    """Get the safetensors parameters metadata for remote/local model repository."""
    if (model_path := Path(model)).exists():
        return _read_safetensors_metadata_in_dir(model_path)

    repo_mt = try_get_safetensors_metadata(model, revision=revision)
    if repo_mt and (files_mt := repo_mt.files_metadata):
        return {
            param_name: asdict(info)
            for file_mt in files_mt.values()
            for param_name, info in file_mt.tensors.items()
        }

    # Hub fetch failed (e.g. 429, network unreachable). Fall back to the
    # local HF cache: weights may already be cached from a prior run, and
    # weight loading itself uses the same cache.
    try:
        local_dir = hf_api().snapshot_download(
            repo_id=model,
            revision=revision,
            allow_patterns=["*.safetensors"],
            local_files_only=True,
        )
    except huggingface_hub.errors.LocalEntryNotFoundError as e:
        logger.warning_once(
            "Could not retrieve safetensors metadata for %s "
            "(Hub fetch failed and no local cache snapshot is available): %s.",
            model,
            str(e),
        )
        return {}
    return _read_safetensors_metadata_in_dir(Path(local_dir))

@cache
def get_sentence_transformer_tokenizer_config(
    model: str | Path, revision: str | None = "main"
) -> dict[str, Any] | None:
    """Returns the tokenization configuration dictionary for a
    given Sentence Transformer BERT model.

    Args:
        model (str|Path): The name of the Sentence Transformer BERT model.
        revision (str, optional): The revision of the model to use. Defaults to 'main'.

    Returns:
    - dict: A dictionary containing the configuration parameters
    for the Sentence Transformer BERT model.

    """
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None

    for config_file in sentence_transformer_config_files:
        if isinstance(
            try_get_local_file(model=model, file_name=config_file, revision=revision),
            Path,
        ):
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
            if encoder_dict:
                break

    if not encoder_dict and not Path(model).is_absolute():
        try:
            # If model is on HuggingfaceHub, get the repo files
            repo_files = list_repo_files(model, revision=revision)
        except Exception:
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
                if encoder_dict:
                    break

    if not encoder_dict:
        return None

    logger.info("Found sentence-transformers tokenize configuration.")

    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None

def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())

def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
    """Check if rope_parameters is nested by layer types."""
    # Cannot be nested if rope_parameters is empty
    if not rope_parameters:
        return False
    return set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES)

def maybe_override_with_speculators(
    model: str,
    tokenizer: str | None,
    trust_remote_code: bool,
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
    hf_token: bool | str | None = None,
    **kwargs,
) -> tuple[str, str | None, dict[str, Any] | None]:
    """Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config
        hf_token: HuggingFace token for authenticated model access

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)

    """
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
    config_dict, _ = PretrainedConfig.get_config_dict(
        model,
        revision=revision,
        token=hf_token,
        **without_trust_remote_code(kwargs),
    )
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig

    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
        config_dict=config_dict
    )
    speculators_method = speculative_config["method"]

    # Apply user --speculative-config overrides (e.g. attention_backend).
    if isinstance(vllm_speculative_config, dict):
        speculative_config.update(vllm_speculative_config)

    # Lock fields dictated by the speculators format
    speculative_config["method"] = speculators_method
    speculative_config["model"] = model

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

    return model, tokenizer, speculative_config

def maybe_register_config_serialize_by_value() -> None:
    """Try to register HF model configuration class to serialize by value

    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.

    Examples:

    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
    try:
        import transformers_modules

        transformers_modules_available = True
    except ImportError:
        transformers_modules_available = False

    try:
        import multiprocessing
        import pickle

        import cloudpickle

        from vllm.config import VllmConfig

        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
        def _reduce_config(config: VllmConfig):
            return (pickle.loads, (cloudpickle.dumps(config),))

        multiprocessing.reducer.register(VllmConfig, _reduce_config)

        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
            from vllm.v1.executor.ray_utils import ray

            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
            exc_info=e,
        )

def mrope_num_dims(config: PretrainedConfig) -> int:
    """Number of M-RoPE position channels the model consumes.

    Each section entry sizes one channel, so the section length is the channel
    count. Interleaved M-RoPE also accepts a 2 section variant whose positions
    are still 3D, so never return fewer than 3.
    """
    if not uses_mrope(config):
        return 0

    for candidate in (config.get_text_config(), config):
        mrope_section = _mrope_section(candidate)
        if mrope_section is not None:
            return max(len(mrope_section), 3)

    # Custom configs may declare M-RoPE without exposing the sections.
    return 3

def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None:
    """Patch legacy RoPE type fields for backwards compatibility with
    older custom models which would otherwise fail to load.
    """
    # No RoPE parameters to patch
    if rope_parameters is None:
        return

    def _patch_legacy_rope_type(rope_parameters: dict[str, Any]) -> None:
        # Case 1: Both legacy and modern fields present - check for conflicts
        if "rope_type" in rope_parameters and "type" in rope_parameters:
            rope_type = rope_parameters["rope_type"]
            rope_type_legacy = rope_parameters["type"]
            if (rope_type_legacy == "su" and rope_type == "longrope") or (
                rope_type_legacy == "mrope" and rope_type == "default"
            ):
                pass  # No action needed
            elif rope_type != rope_type_legacy:
                raise ValueError(
                    f"Found conflicts between 'rope_type={rope_type}' (modern "
                    f"field) and 'type={rope_type_legacy}' (legacy field). "
                    "You should only specify one of them."
                )
        # Case 2: Only legacy field present - patch to modern format with warning
        if "rope_type" not in rope_parameters and "type" in rope_parameters:
            rope_parameters["rope_type"] = rope_parameters["type"]
            logger.info("Replacing legacy 'type' key with 'rope_type'")
        # Case 3: No rope_type field present - nothing to patch
        if "rope_type" not in rope_parameters:
            return
        # Patch legacy rope_type values with warning
        if rope_parameters["rope_type"] == "su":
            rope_parameters["rope_type"] = "longrope"
            logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
        elif rope_parameters["rope_type"] == "mrope":
            if "mrope_section" not in rope_parameters:
                raise ValueError(
                    "Legacy rope_type 'mrope' requires "
                    "'mrope_section' in rope_parameters"
                )
            rope_parameters["rope_type"] = "default"
            logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
        elif rope_parameters["rope_type"] == "telechat3-yarn":
            # TeleChat3 is YaRN with 0.07 in place of YaRN's 0.1 attention
            # scaling coefficient. Precompute it so the config is plain YaRN.
            factor = rope_parameters["factor"]
            rope_parameters["rope_type"] = "yarn"
            rope_parameters["attention_factor"] = 0.07 * math.log(factor) + 1.0
            rope_parameters.pop("type", None)
            logger.warning("Replacing rope_type 'telechat3-yarn' with 'yarn'")

    # Handle nested rope_parameters in interleaved sliding attention models
    if is_rope_parameters_nested(rope_parameters):
        for rope_parameters_layer_type in rope_parameters.values():
            if rope_parameters_layer_type is not None:
                _patch_legacy_rope_type(rope_parameters_layer_type)
    else:
        _patch_legacy_rope_type(rope_parameters)

def patch_rope_parameters(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    from vllm.config.utils import getattr_iter

    # Older custom models may use non-standard field names which need patching.
    names = ["rope_theta", "rotary_emb_base"]
    rope_theta = getattr_iter(config, names, None, warn=True)
    names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
    partial_rotary_factor = getattr_iter(config, names, None, warn=True)

    if rope_theta is not None or getattr(config, "rope_parameters", None):
        # Patch these fields in case they used non-standard names
        if rope_theta is not None:
            config.rope_theta = rope_theta
        if partial_rotary_factor is not None:
            config.partial_rotary_factor = partial_rotary_factor
        # Standardize and validate RoPE parameters
        patch_legacy_rope_type(getattr(config, "rope_parameters", None))
        config.standardize_rope_params()
        config.validate_rope()

def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.

    Args:
         config_format (str): The config parser format name.

    Examples:
         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>

    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
        if not issubclass(config_parser_cls, ConfigParserBase):
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
        return config_parser_cls

    return _wrapper

def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing.
    """
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta

def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)

def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )