diff --git a/common/common.cpp b/common/common.cpp index ac2de234..9f1d3a51 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -3023,6 +3023,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "speculative", "-tbd, --threads-batch-draft N", "number of threads to use during batch and prompt processing (default: same as --threads-draft)" }); options.push_back({ "speculative", "-ps, --p-split N", "speculative decoding split probability (default: %.1f)", (double)params.p_split }); + options.push_back({ "speculative", " --spec-replace TARGET DRAFT","replace TARGET token with DRAFT token in speculative decoding" }); options.push_back({ "*", "-lcs, --lookup-cache-static FNAME", "path to static lookup cache to use for lookup decoding (not updated by generation)" }); options.push_back({ "*", "-lcd, --lookup-cache-dynamic FNAME", @@ -3095,6 +3096,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "main infill", " --in-prefix-bos", "prefix BOS to user inputs, preceding the `--in-prefix` string" }); options.push_back({ "main infill", " --in-prefix STRING", "string to prefix user inputs with (default: empty)" }); options.push_back({ "main infill", " --in-suffix STRING", "string to suffix after user inputs with (default: empty)" }); + options.push_back({ "main infill", " --infill", "use infill mode" }); options.push_back({ "main", " --no-warmup", "skip warming up the model with an empty run" }); options.push_back({ "server infill", " --spm-infill", "use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: %s)", params.spm_infill ? "enabled" : "disabled" }); @@ -3125,6 +3127,11 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "*", " --mirostat-ent N", "Mirostat target entropy, parameter tau (default: %.1f)", (double)sparams.mirostat_tau }); options.push_back({ "*", " --xtc-probability p", "xtc probability (default: %.1f, 0.0 = disabled)", (double)sparams.xtc_probability }); options.push_back({ "*", " --xtc-threshold t", "xtc threshold (default: %.1f, >0.5 = disabled)", (double)sparams.xtc_threshold}); + options.push_back({ "*", " --dry-multiplier N", "DRY sampling multiplier (default: %.1f, 0.0 = disabled)", (double)sparams.dry_multiplier }); + options.push_back({ "*", " --dry-base N", "DRY sampling base (default: %.2f)", (double)sparams.dry_base }); + options.push_back({ "*", " --dry-allowed-length N", "DRY sampling allowed length (default: %d)", sparams.dry_allowed_length }); + options.push_back({ "*", " --dry-penalty-last-n N", "DRY sampling penalty last N tokens (default: %d, 0 = disabled, -1 = context size)", sparams.dry_penalty_last_n }); + options.push_back({ "*", " --dry-sequence-breaker STR", "DRY sampling sequence breaker characters (each char becomes a breaker) or 'none' to clear" }); options.push_back({ "*", " --top-n-sigma t", "top-n-sigma parmeter (default: %.1f, 0.0 = disabled)", (double)sparams.top_n_sigma}); options.push_back({ "*", " --adaptive-target", "adaptive-p sampling: (default: %.2f, <0.0 = disabled)", (double)sparams.adaptive_target}); options.push_back({ "*", " --adaptive-decay", "adaptive-p sampling: (default: %.2f)", (double)sparams.adaptive_decay}); @@ -3246,6 +3253,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "multi-modality" }); options.push_back({ "*", " --mmproj FILE", "path to a multimodal projector file. see examples/mtmd/README.md" }); options.push_back({ "*", " --image FILE", "path to an image file. use with multimodal models. Specify multiple times for batching" }); + options.push_back({ "*", " --audio FILE", "path to an audio file for multimodal models. Specify multiple times for batching" }); + options.push_back({ "*", " --mmproj-url URL", "URL to download the multimodal projector file" }); + options.push_back({ "*", " --no-mmproj-offload", "do not offload multimodal projector to GPU (default: offload enabled)" }); options.push_back({ "*", " --image-min-tokens N", "minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)"}); options.push_back({ "*", " --image-max-tokens N", "maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)" }); options.push_back({ "*", " --mtmd-kq-type TYPE", "data type for multimodality K*Q (default: %s)", params.mtmd_kq_type.c_str() }); @@ -3269,6 +3279,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "*", " --prefetch-experts-threads N", "number of expert prefetch workers, tune to drive speed/type (default: auto)"}); options.push_back({ "*", " --fit-margin N", "safety margin in MiB when auto-fitting model offloading"}); + options.push_back({ "*", "-gfm, --gpu-fit-margin N", "per-layer GPU fit margin as layer_id,margin pairs, comma-separated" }); options.push_back({ "*", "-wgt, --worst-graph-tokens N", "number of tokens to use for worst-case graph"}); options.push_back({ "*", " --fit", "automatically determine which tensors to offload to the GPU(s)"}); options.push_back({ "*", " --numa TYPE", "attempt optimizations that help on some NUMA systems\n" @@ -3296,6 +3307,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "*", "-devd, --device-draft dev1,dev2", "comma-separated list of devices to use for offloading for the draft model (none = don't offload)\n" "Example: CUDA0,CUDA1,RPC[192.168.0.1:8080]\n" }); + options.push_back({ "*", "-op, --offload-policy POLICY","set per-layer offload policy as layer_id,0|1 pairs, comma-separated" }); + options.push_back({ "*", "-no-ooae, --no-offload-only-active-experts", + "do not offload only active experts" }); options.push_back({ "*", "-mg, --main-gpu i", "the GPU to use for the model (with split-mode = none),\n" "or for intermediate results and KV (with split-mode = row) (default: %d)", params.main_gpu }); options.push_back({ "*", "--max-gpu i", "max. number of GPUs to use at a time with split mode 'graph', (default: %d)", params.max_gpu }); @@ -3303,6 +3317,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "model" }); options.push_back({ "*", " --check-tensors", "check model tensor data for invalid values (default: %s)", params.check_tensors ? "true" : "false" }); + options.push_back({ "*", "-ot, --override-tensor NAME", "override tensor buffer type as tensor_name=buft, comma-separated" }); options.push_back({ "*", " --override-kv KEY=TYPE:VALUE", "advanced option to override model metadata by key. may be specified multiple times.\n" "types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false" }); @@ -3358,12 +3373,16 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "imatrix", " --process-output", "collect data for the output tensor (default: %s)", params.process_output ? "true" : "false" }); options.push_back({ "imatrix", " --no-ppl", "do not compute perplexity (default: %s)", params.compute_ppl ? "true" : "false" }); options.push_back({ "imatrix", " --chunk N", "start processing the input from chunk N (default: %d)", params.i_chunk }); + options.push_back({ "imatrix", " --output-tensor-name FNAME", "output tensor name for imatrix data (default: '%s')", params.output_tensor_name.c_str() }); options.push_back({ "bench" }); options.push_back({ "bench", "-pps", "is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false" }); options.push_back({ "bench", "-npp n0,n1,...", "number of prompt tokens" }); options.push_back({ "bench", "-ntg n0,n1,...", "number of text generation tokens" }); options.push_back({ "bench", "-npl n0,n1,...", "number of parallel prompts" }); + options.push_back({ "bench", "-nrep, --n-repetitions N", "number of repetitions (default: %d)", params.nrep }); + options.push_back({ "bench", "-wb, --warmup-batch", "run a warmup batch before measurement" }); + options.push_back({ "bench", " --output-format FORMAT", "output format: table, jsonl, or csv (default: table)" }); options.push_back({ "embedding" }); options.push_back({ "embedding", " --embd-normalize", "normalisation for embendings (default: %d) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)", params.embd_normalize }); @@ -3402,6 +3421,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "server", "-sps, --slot-prompt-similarity SIMILARITY", "how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity }); options.push_back({ "server", " --lora-init-without-apply", "load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: %s)", params.lora_init_without_apply ? "enabled" : "disabled"}); + options.push_back({ "server", " --send-done", "send 'done' signal to the client when generation is complete" }); + options.push_back({ "server", " --sql-save-file FNAME", "save chat history to a SQLite database" }); + options.push_back({ "server", " --sqlite-zstd-ext-file FNAME", "path to SQLite ZSTD extension for compression" }); #ifndef LOG_DISABLE_LOGS options.push_back({ "logging" }); diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index b3d0a4fe..868488af 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -224,6 +224,8 @@ static llama_init_result ik_init_from_loaded_model(llama_model * model, gpt_para static void print_usage(int argc, char ** argv, const gpt_params & params) { gpt_params_print_usage(argc, argv, params); + LOG_TEE("\nimatrix-specific options:\n\n"); + LOG_TEE(" -lsim, --layer-similarity collect layer similarity data\n"); LOG_TEE("\nexample usage:\n"); LOG_TEE("\n %s \\\n" " -m model.gguf -f some-text.txt [-o imatrix.dat] [--process-output] [--verbosity 1] \\\n" diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 621b5adf..7ff87e18 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -151,7 +151,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--per-layer-token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); @@ -162,7 +162,8 @@ static void usage(const char * executable) { printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor.\n"); - printf(" --token-embedding-type ggml_type: use this ggml_type for the token_embd.weight tensor.\n\n"); + printf(" --token-embedding-type ggml_type: use this ggml_type for the token_embd.weight tensor.\n"); + printf(" --per-layer-token-embedding-type ggml_type: use this ggml_type for all per-layer token_embd.weight tensors.\n\n"); printf(" --extra-output-tensor ggml_type: requantize and add output tensor of that type.\n"); printf(" --ffn-gate-inp-type ggml_type: use this ggml_type for the ffn_gate_inp tensors.\n\n"); printf(" --custom-q regex1=type1,regex2=type2...: use this to specify custom quantization type rules.\n\n"); diff --git a/examples/sweep-bench/sweep-bench.cpp b/examples/sweep-bench/sweep-bench.cpp index fe44fb6d..9c83c43c 100644 --- a/examples/sweep-bench/sweep-bench.cpp +++ b/examples/sweep-bench/sweep-bench.cpp @@ -56,7 +56,13 @@ static void llama_selective_log_callback(ggml_log_level level, const char * text LOG_TEE("%s", text); } -static void print_usage(int, char ** argv) { +static void print_usage(int argc, char ** argv) { + gpt_params_print_usage(argc, argv, gpt_params()); + + LOG_TEE("\nsweep-bench specific options:\n\n"); + LOG_TEE(" -nrep, --n-repetitions N number of repetitions for each context size (default: 1)\n"); + LOG_TEE(" -wb, --warmup-batch run a warmup batch before measurement\n"); + LOG_TEE(" --output-format FORMAT output format: table (default) or jsonl\n"); LOG_TEE("\nexample usage:\n"); LOG_TEE("\n %s -m model.gguf -c 8192 -b 2048 -ub 512\n", argv[0]); LOG_TEE("\n");