aditya/llama.cpp
1
0
Fork 0
mirror of https://git.adityakumar.xyz/llama.cpp.git synced 2024-11-09 23:29:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove default arguments from sampling functions (#1343)

Browse source
This commit is contained in:
Jed Fox 2023-05-06 17:01:47 -04:00 committed by GitHub
parent 173d0e6419
commit 3924088512
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 14 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -21,6 +21,7 @@ build-sanitize-addr/
build-sanitize-thread/ build-sanitize-thread/
models/* models/*
*.bin
/main /main
/quantize /quantize

8
examples/main/main.cpp
View file

@ -444,10 +444,10 @@ int main(int argc, char ** argv) {
id = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); id = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
} else { } else {
// Temperature sampling // Temperature sampling
llama_sample_top_k(ctx, &candidates_p, top_k); llama_sample_top_k(ctx, &candidates_p, top_k, 1);
llama_sample_tail_free(ctx, &candidates_p, tfs_z); llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1);
llama_sample_typical(ctx, &candidates_p, typical_p); llama_sample_typical(ctx, &candidates_p, typical_p, 1);
llama_sample_top_p(ctx, &candidates_p, top_p); llama_sample_top_p(ctx, &candidates_p, top_p, 1);
llama_sample_temperature(ctx, &candidates_p, temp); llama_sample_temperature(ctx, &candidates_p, temp);
id = llama_sample_token(ctx, &candidates_p); id = llama_sample_token(ctx, &candidates_p);
} }

2
llama.cpp
View file

@ -1791,7 +1791,7 @@ llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_
float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat); float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat);
// Sample the next word X using top-k sampling // Sample the next word X using top-k sampling
llama_sample_top_k(nullptr, candidates, int(k)); llama_sample_top_k(nullptr, candidates, int(k), 1);
if (ctx) { if (ctx) {
ctx->t_sample_us += ggml_time_us() - t_start_sample_us; ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
} }

8
llama.h
View file

@ -202,16 +202,16 @@ extern "C" {
LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates); LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates);
/// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep = 1); LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep);
/// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep = 1); LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep);
/// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep = 1); LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep);
/// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep = 1); LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep);
LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp); LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp);
/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

8
tests/test-sampling.cpp
View file

@ -32,7 +32,7 @@ void test_top_k(const std::vector<float> & probs,
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
llama_sample_softmax(nullptr, &candidates_p); llama_sample_softmax(nullptr, &candidates_p);
DUMP(&candidates_p); DUMP(&candidates_p);
llama_sample_top_k(nullptr, &candidates_p, k); llama_sample_top_k(nullptr, &candidates_p, k, 1);
DUMP(&candidates_p); DUMP(&candidates_p);
assert(candidates_p.size == expected_probs.size()); assert(candidates_p.size == expected_probs.size());
@ -57,7 +57,7 @@ void test_top_p(const std::vector<float> & probs,
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
llama_sample_softmax(nullptr, &candidates_p); llama_sample_softmax(nullptr, &candidates_p);
DUMP(&candidates_p); DUMP(&candidates_p);
llama_sample_top_p(nullptr, &candidates_p, p); llama_sample_top_p(nullptr, &candidates_p, p, 1);
DUMP(&candidates_p); DUMP(&candidates_p);
assert(candidates_p.size == expected_probs.size()); assert(candidates_p.size == expected_probs.size());
@ -80,7 +80,7 @@ void test_tfs(const std::vector<float> & probs,
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
DUMP(&candidates_p); DUMP(&candidates_p);
llama_sample_tail_free(nullptr, &candidates_p, z); llama_sample_tail_free(nullptr, &candidates_p, z, 1);
DUMP(&candidates_p); DUMP(&candidates_p);
assert(candidates_p.size == expected_probs.size()); assert(candidates_p.size == expected_probs.size());
@ -103,7 +103,7 @@ void test_typical(const std::vector<float> & probs,
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
DUMP(&candidates_p); DUMP(&candidates_p);
llama_sample_typical(nullptr, &candidates_p, p); llama_sample_typical(nullptr, &candidates_p, p, 1);
DUMP(&candidates_p); DUMP(&candidates_p);
assert(candidates_p.size == expected_probs.size()); assert(candidates_p.size == expected_probs.size());