mirror of
https://git.adityakumar.xyz/llama.cpp.git
synced 2024-11-13 00:39:47 +00:00
563cdc391d
* Support calling mlock() on loaded model data on Linux and macOS This is enabled by a new --mlock command line option. Using mlock() disables swapping and memory compression for the model data. Doing so can be useful on systems where the model takes up a large fraction of system RAM. In my experience, macOS is quite eager to start compressing llama.cpp's memory, which then makes it halt for a few seconds while it decompresses, even with a model that uses "only" 25GB out of 32GB. Of course, this comes at the cost of forcing the system to swap or compress other processes' memory instead, so it needs to be used with care and shouldn't be enabled by default. In theory it should be possible to support this on Windows as well using VirtualLock(), but I'm not much of a Windows user. * Update llama.cpp --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
237 lines
9.7 KiB
C++
237 lines
9.7 KiB
C++
#include "ggml.h"
|
|
|
|
#include "utils.h"
|
|
|
|
#include <cassert>
|
|
#include <cstring>
|
|
#include <fstream>
|
|
#include <string>
|
|
#include <iterator>
|
|
#include <algorithm>
|
|
|
|
#if defined(_MSC_VER) || defined(__MINGW32__)
|
|
#include <malloc.h> // using malloc.h with MSC/MINGW
|
|
#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
|
|
#include <alloca.h>
|
|
#endif
|
|
|
|
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
|
// determine sensible default number of threads.
|
|
// std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
|
|
#ifdef __linux__
|
|
std::ifstream cpuinfo("/proc/cpuinfo");
|
|
params.n_threads = std::count(std::istream_iterator<std::string>(cpuinfo),
|
|
std::istream_iterator<std::string>(),
|
|
std::string("processor"));
|
|
#endif
|
|
if (params.n_threads == 0) {
|
|
params.n_threads = std::max(1, (int32_t) std::thread::hardware_concurrency());
|
|
}
|
|
|
|
bool invalid_param = false;
|
|
std::string arg;
|
|
for (int i = 1; i < argc; i++) {
|
|
arg = argv[i];
|
|
|
|
if (arg == "-s" || arg == "--seed") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.seed = std::stoi(argv[i]);
|
|
} else if (arg == "-t" || arg == "--threads") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.n_threads = std::stoi(argv[i]);
|
|
} else if (arg == "-p" || arg == "--prompt") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.prompt = argv[i];
|
|
} else if (arg == "-f" || arg == "--file") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
std::ifstream file(argv[i]);
|
|
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
|
|
if (params.prompt.back() == '\n') {
|
|
params.prompt.pop_back();
|
|
}
|
|
} else if (arg == "-n" || arg == "--n_predict") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.n_predict = std::stoi(argv[i]);
|
|
} else if (arg == "--top_k") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.top_k = std::stoi(argv[i]);
|
|
} else if (arg == "-c" || arg == "--ctx_size") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.n_ctx = std::stoi(argv[i]);
|
|
} else if (arg == "--memory_f16") {
|
|
params.memory_f16 = true;
|
|
} else if (arg == "--top_p") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.top_p = std::stof(argv[i]);
|
|
} else if (arg == "--temp") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.temp = std::stof(argv[i]);
|
|
} else if (arg == "--repeat_last_n") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.repeat_last_n = std::stoi(argv[i]);
|
|
} else if (arg == "--repeat_penalty") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.repeat_penalty = std::stof(argv[i]);
|
|
} else if (arg == "-b" || arg == "--batch_size") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.n_batch = std::stoi(argv[i]);
|
|
} else if (arg == "-m" || arg == "--model") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.model = argv[i];
|
|
} else if (arg == "-i" || arg == "--interactive") {
|
|
params.interactive = true;
|
|
} else if (arg == "--embedding") {
|
|
params.embedding = true;
|
|
} else if (arg == "--interactive-start") {
|
|
params.interactive = true;
|
|
} else if (arg == "--interactive-first") {
|
|
params.interactive_start = true;
|
|
} else if (arg == "-ins" || arg == "--instruct") {
|
|
params.instruct = true;
|
|
} else if (arg == "--color") {
|
|
params.use_color = true;
|
|
} else if (arg == "--mlock") {
|
|
params.use_mlock = true;
|
|
} else if (arg == "-r" || arg == "--reverse-prompt") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.antiprompt.push_back(argv[i]);
|
|
} else if (arg == "--perplexity") {
|
|
params.perplexity = true;
|
|
} else if (arg == "--ignore-eos") {
|
|
params.ignore_eos = true;
|
|
} else if (arg == "--n_parts") {
|
|
if (++i >= argc) {
|
|
invalid_param = true;
|
|
break;
|
|
}
|
|
params.n_parts = std::stoi(argv[i]);
|
|
} else if (arg == "-h" || arg == "--help") {
|
|
gpt_print_usage(argc, argv, params);
|
|
exit(0);
|
|
} else if (arg == "--random-prompt") {
|
|
params.random_prompt = true;
|
|
} else {
|
|
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
|
gpt_print_usage(argc, argv, params);
|
|
exit(1);
|
|
}
|
|
}
|
|
if (invalid_param) {
|
|
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
|
gpt_print_usage(argc, argv, params);
|
|
exit(1);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
|
|
fprintf(stderr, "usage: %s [options]\n", argv[0]);
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "options:\n");
|
|
fprintf(stderr, " -h, --help show this help message and exit\n");
|
|
fprintf(stderr, " -i, --interactive run in interactive mode\n");
|
|
fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n");
|
|
fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n");
|
|
fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n");
|
|
fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n");
|
|
fprintf(stderr, " specified more than once for multiple prompts).\n");
|
|
fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n");
|
|
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n");
|
|
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
|
|
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
|
|
fprintf(stderr, " prompt to start generation with (default: empty)\n");
|
|
fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
|
|
fprintf(stderr, " -f FNAME, --file FNAME\n");
|
|
fprintf(stderr, " prompt file to start generation.\n");
|
|
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict);
|
|
fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k);
|
|
fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p);
|
|
fprintf(stderr, " --repeat_last_n N last n tokens to consider for penalize (default: %d)\n", params.repeat_last_n);
|
|
fprintf(stderr, " --repeat_penalty N penalize repeat sequence of tokens (default: %.1f)\n", params.repeat_penalty);
|
|
fprintf(stderr, " -c N, --ctx_size N size of the prompt context (default: %d)\n", params.n_ctx);
|
|
fprintf(stderr, " --ignore-eos ignore end of stream token and continue generating\n");
|
|
fprintf(stderr, " --memory_f16 use f16 instead of f32 for memory key+value\n");
|
|
fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp);
|
|
fprintf(stderr, " --n_parts N number of model parts (default: -1 = determine from dimensions)\n");
|
|
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
|
|
fprintf(stderr, " --perplexity compute perplexity over the prompt\n");
|
|
if (ggml_mlock_supported()) {
|
|
fprintf(stderr, " --mlock force system to keep model in RAM rather than swapping or compressing\n");
|
|
}
|
|
fprintf(stderr, " -m FNAME, --model FNAME\n");
|
|
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
|
|
fprintf(stderr, "\n");
|
|
}
|
|
|
|
std::string gpt_random_prompt(std::mt19937 & rng) {
|
|
const int r = rng() % 10;
|
|
switch (r) {
|
|
case 0: return "So";
|
|
case 1: return "Once upon a time";
|
|
case 2: return "When";
|
|
case 3: return "The";
|
|
case 4: return "After";
|
|
case 5: return "If";
|
|
case 6: return "import";
|
|
case 7: return "He";
|
|
case 8: return "She";
|
|
case 9: return "They";
|
|
default: return "To";
|
|
}
|
|
|
|
return "The";
|
|
}
|
|
|
|
// TODO: not great allocating this every time
|
|
std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) {
|
|
// initialize to prompt numer of chars, since n_tokens <= n_prompt_chars
|
|
std::vector<llama_token> res(text.size() + (int)add_bos);
|
|
int n = llama_tokenize(ctx, text.c_str(), res.data(), res.size(), add_bos);
|
|
assert(n >= 0);
|
|
res.resize(n);
|
|
|
|
return res;
|
|
}
|