diff --git a/configs/alpaca-native-enhanced.txt b/configs/alpaca-native-enhanced.txt deleted file mode 100644 index 109d315..0000000 --- a/configs/alpaca-native-enhanced.txt +++ /dev/null @@ -1,21 +0,0 @@ ---ctx_size 2048 ---batch_size 16 ---repeat_penalty 1.15 ---temp 0.4 ---top_k 30 ---top_p 0.18 - ---interactive-first ---keep -1 - ---ins-prefix-bos ---ins-prefix "\n\nUser: " ---ins-suffix "\n\nAssistant: " ---reverse-prompt "User: " - --p "You are an AI language model designed to assist the User by answering their questions, offering advice, and engaging in casual conversation in a friendly, helpful, and informative manner. You respond clearly, coherently, and you consider the conversation history. - -User: Hey, how's it going? - -Assistant: Hey there! I'm doing great, thank you. What can I help you with today? Let's have a fun chat!" - diff --git a/configs/alpaca.txt b/configs/alpaca.txt deleted file mode 100644 index 99a3ab4..0000000 --- a/configs/alpaca.txt +++ /dev/null @@ -1,9 +0,0 @@ ---clean-interface ---interactive-first ---keep -1 ---ins-prefix-bos ---ins-prefix "\n\n### Instruction:\n\n" ---ins-suffix "\n\n### Response:\n\n" ---reverse-prompt "### Instruction:\n\n" - --p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n" diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt deleted file mode 100644 index 0caa749..0000000 --- a/configs/chat-with-bob.txt +++ /dev/null @@ -1,15 +0,0 @@ ---interactive-first ---keep -1 ---ins-prefix-bos ---ins-prefix "\nUser: " ---ins-suffix "\nBob: " ---reverse-prompt "User: " ---rm-trailing-space-workaround - --p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. - -User: Hello, Bob. -Bob: Hello. How may I help you today? -User: Please tell me the largest city in Europe. -Bob: Sure. The largest city in Europe is Moscow, the capital of Russia." - diff --git a/configs/llama.txt b/configs/llama.txt deleted file mode 100644 index 9d23e75..0000000 --- a/configs/llama.txt +++ /dev/null @@ -1,3 +0,0 @@ ---interactive-first ---keep -1 ---temp 0.1 diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt deleted file mode 100644 index efa60d9..0000000 --- a/configs/vicuna-simple.txt +++ /dev/null @@ -1,7 +0,0 @@ ---interactive-first ---keep -1 ---ins-prefix-bos ---ins-prefix "\n### Human: " ---ins-suffix "\n### Assistant: " ---reverse-prompt "### Human: " ---rm-trailing-space-workaround diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt deleted file mode 100644 index 911d067..0000000 --- a/configs/vicuna-stop.txt +++ /dev/null @@ -1,8 +0,0 @@ ---interactive-first ---keep -1 ---ins-prefix-bos ---ins-prefix "\n### Human: " ---ins-suffix "\n### Assistant: " ---reverse-prompt "### Human: " ---stop-prompt "### Assistant: " ---rm-trailing-space-workaround diff --git a/configs/vicuna.txt b/configs/vicuna.txt deleted file mode 100644 index 6d81141..0000000 --- a/configs/vicuna.txt +++ /dev/null @@ -1,9 +0,0 @@ ---interactive-first ---keep -1 ---ins-prefix-bos ---ins-prefix "\n### Human: " ---ins-suffix "\n### Assistant: " ---reverse-prompt "### Human: " ---rm-trailing-space-workaround - --p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." diff --git a/examples/common.cpp b/examples/common.cpp index eaa5ace..0772dbf 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -2,13 +2,10 @@ #include #include -#include #include -#include #include #include #include -#include #if defined (_WIN32) #include @@ -26,43 +23,6 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int #define CP_UTF8 65001 #endif -void split_args(const std::string & args_string, std::vector & output_args) -{ - std::string current_arg = ""; - bool in_quotes = false; - char quote_type; - - for (char c : args_string) { - if (c == '"' || c == '\'') { - if (!in_quotes) { - in_quotes = true; - quote_type = c; - } else if (quote_type == c) { - in_quotes = false; - } else { - current_arg += c; - } - } else if (in_quotes) { - current_arg += c; - } else if (std::isspace(c)) { - if (current_arg != "") { - output_args.push_back(current_arg); - current_arg = ""; - } - } else { - current_arg += c; - } - } - - if (current_arg != "") { - output_args.push_back(current_arg); - } -} - -std::string unescape(const std::string & str) { - return std::regex_replace(str, std::regex("\\\\n"), "\n"); -} - bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { // determine sensible default number of threads. // std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0. @@ -80,11 +40,28 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { std::string arg; gpt_params default_params; - // get additional arguments from config files - std::vector args; for (int i = 1; i < argc; i++) { arg = argv[i]; - if (arg == "--config") { + + if (arg == "-s" || arg == "--seed") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.seed = std::stoi(argv[i]); + } else if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads = std::stoi(argv[i]); + } else if (arg == "-p" || arg == "--prompt") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.prompt = argv[i]; + } else if (arg == "-f" || arg == "--file") { if (++i >= argc) { invalid_param = true; break; @@ -95,153 +72,85 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - std::string args_string; - std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(args_string)); - if (args_string.back() == '\n') { - args_string.pop_back(); - } - split_args(args_string, args); - for (int j = 0; j < args.size(); j++) { - args[j] = unescape(args[j]); - } - } else { - args.emplace_back(argv[i]); - } - } - - // parse args - int args_c = static_cast(args.size()); - for (int i = 0; i < args_c && !invalid_param; i++) { - arg = args[i]; - - if (arg == "-s" || arg == "--seed") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.seed = std::stoi(args[i]); - } else if (arg == "-t" || arg == "--threads") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.n_threads = std::stoi(args[i]); - } else if (arg == "-p" || arg == "--prompt") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.prompt = args[i]; - } else if (arg == "-f" || arg == "--file") { - if (++i >= args_c) { - invalid_param = true; - break; - } - std::ifstream file(args[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str()); - invalid_param = true; - break; - } std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); if (params.prompt.back() == '\n') { params.prompt.pop_back(); } } else if (arg == "-n" || arg == "--n_predict") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.n_predict = std::stoi(args[i]); + params.n_predict = std::stoi(argv[i]); } else if (arg == "--top_k") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.top_k = std::stoi(args[i]); + params.top_k = std::stoi(argv[i]); } else if (arg == "-c" || arg == "--ctx_size") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.n_ctx = std::stoi(args[i]); + params.n_ctx = std::stoi(argv[i]); } else if (arg == "--memory_f32") { params.memory_f16 = false; } else if (arg == "--top_p") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.top_p = std::stof(args[i]); + params.top_p = std::stof(argv[i]); } else if (arg == "--temp") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.temp = std::stof(args[i]); + params.temp = std::stof(argv[i]); } else if (arg == "--repeat_last_n") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.repeat_last_n = std::stoi(args[i]); + params.repeat_last_n = std::stoi(argv[i]); } else if (arg == "--repeat_penalty") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.repeat_penalty = std::stof(args[i]); + params.repeat_penalty = std::stof(argv[i]); } else if (arg == "-b" || arg == "--batch_size") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.n_batch = std::stoi(args[i]); + params.n_batch = std::stoi(argv[i]); params.n_batch = std::min(512, params.n_batch); } else if (arg == "--keep") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.n_keep = std::stoi(args[i]); + params.n_keep = std::stoi(argv[i]); } else if (arg == "-m" || arg == "--model") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.model = args[i]; + params.model = argv[i]; } else if (arg == "-i" || arg == "--interactive") { params.interactive = true; } else if (arg == "--embedding") { params.embedding = true; - } else if (arg == "--clean-interface") { - params.clean_interface = true; } else if (arg == "--interactive-start") { params.interactive = true; } else if (arg == "--interactive-first") { params.interactive_start = true; } else if (arg == "-ins" || arg == "--instruct") { - fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n" - "--clean-interface " - "--interactive-first " - "--keep -1 " - "--ins-prefix-bos " - "--ins-prefix \"\\n\\n### Instruction:\\n\\n\" " - "--ins-suffix \"\\n\\n### Response:\\n\\n\" " - "-r \"### Instruction:\\n\\n\" " - "\n\n"); - // params.instruct = true; - params.clean_interface = true; - params.interactive_start = true; - params.n_keep = -1; - params.instruct_prefix_bos = true; - params.instruct_prefix = "\n\n### Instruction:\n\n"; - params.instruct_suffix = "\n\n### Response:\n\n"; - params.antiprompt.push_back("### Instruction:\n\n"); + params.instruct = true; } else if (arg == "--color") { params.use_color = true; - } else if (arg == "--disable-multiline") { - params.multiline_mode = false; } else if (arg == "--mlock") { params.use_mlock = true; } else if (arg == "--no-mmap") { @@ -251,94 +160,65 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { } else if (arg == "--verbose-prompt") { params.verbose_prompt = true; } else if (arg == "-r" || arg == "--reverse-prompt") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.antiprompt.push_back(args[i]); - } else if (arg == "--stop-prompt") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.stopprompt.push_back(args[i]); - } else if (arg == "--rm-trailing-space-workaround") { - params.rm_trailing_space_workaround = true; + params.antiprompt.push_back(argv[i]); } else if (arg == "--perplexity") { params.perplexity = true; } else if (arg == "--ignore-eos") { params.ignore_eos = true; } else if (arg == "--n_parts") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.n_parts = std::stoi(args[i]); + params.n_parts = std::stoi(argv[i]); } else if (arg == "-h" || arg == "--help") { - gpt_print_usage(argv[0], default_params); + gpt_print_usage(argc, argv, default_params); exit(0); } else if (arg == "--random-prompt") { params.random_prompt = true; } else if (arg == "--in-prefix") { - if (++i >= args_c) { + if (++i >= argc) { invalid_param = true; break; } - params.input_prefix = args[i]; - } else if (arg == "--ins-prefix-bos") { - params.instruct_prefix_bos = true; - } else if (arg == "--ins-prefix") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.instruct_prefix = args[i]; - } else if (arg == "--ins-suffix-bos") { - params.instruct_suffix_bos = true; - } else if (arg == "--ins-suffix") { - if (++i >= args_c) { - invalid_param = true; - break; - } - params.instruct_suffix = args[i]; + params.input_prefix = argv[i]; } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - gpt_print_usage(argv[0], default_params); + gpt_print_usage(argc, argv, default_params); exit(1); } } if (invalid_param) { fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); - gpt_print_usage(argv[0], default_params); + gpt_print_usage(argc, argv, default_params); exit(1); } return true; } -void gpt_print_usage(char * argv_0, const gpt_params & params) { - fprintf(stderr, "usage: %s [options]\n", argv_0); +void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "options:\n"); fprintf(stderr, " -h, --help show this help message and exit\n"); fprintf(stderr, " -i, --interactive run in interactive mode\n"); fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n"); - fprintf(stderr, " --clean-interface hides input prefix & suffix and displays '>' instead\n"); + fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n"); fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n"); fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); - fprintf(stderr, " --disable-multiline disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); fprintf(stderr, " prompt to start generation with (default: empty)\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); - fprintf(stderr, " --ins-prefix STRING (instruct) prefix user inputs with tokenized string (default: empty)\n"); - fprintf(stderr, " --ins-prefix-bos (instruct) prepend bos token to instruct prefix.\n"); - fprintf(stderr, " --ins-suffix STRING (instruct) suffix user inputs with tokenized string (default: empty)\n"); - fprintf(stderr, " --ins-suffix-bos (instruct) prepend bos token to instruct suffix.\n"); fprintf(stderr, " -f FNAME, --file FNAME\n"); fprintf(stderr, " prompt file to start generation.\n"); fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict); @@ -448,61 +328,3 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) { str = strTo; } #endif - -bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) { - bool another_line = true; - bool is_eof_multiline_toggled = false; - do { - std::string line; -#if defined(_WIN32) - auto & stdcin = std::wcin; - std::wstring wline; - if (!std::getline(stdcin, wline)) { - // input stream is bad or EOF received - if (stdcin.bad()) { - fprintf(stderr, "%s: error: input stream bad\n", __func__); - return 1; - } - } - win32_utf8_encode(wline, line); -#else - auto & stdcin = std::cin; - if (!std::getline(stdcin, line)) { - // input stream is bad or EOF received - if (stdcin.bad()) { - fprintf(stderr, "%s: error: input stream bad\n", __func__); - return 1; - } - } -#endif - if (stdcin.eof()) { - stdcin.clear(); - stdcin.seekg(0, std::ios::beg); - if (!eof_toggled_multiline_mode) { - another_line = false; - } else { - is_eof_multiline_toggled = !is_eof_multiline_toggled; - if (is_eof_multiline_toggled) { - input_text += line; - continue; - } - } - } - if (!eof_toggled_multiline_mode) { - if (line.empty() || line.back() != '\\') { - another_line = false; - } else { - line.pop_back(); // Remove the continue character - } - } else { - if (!is_eof_multiline_toggled) { - another_line = false; - } - } - input_text += line; - if (another_line) { - input_text += '\n'; // Append the line to the result - } - } while (another_line); - return true; -} diff --git a/examples/common.h b/examples/common.h index df8e4c6..1ea6f74 100644 --- a/examples/common.h +++ b/examples/common.h @@ -14,14 +14,14 @@ // struct gpt_params { - int32_t seed = -1; // RNG seed - int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default) - int32_t n_predict = 128; // new tokens to predict - int32_t repeat_last_n = 64; // last n tokens to penalize - int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions) - int32_t n_ctx = 512; // context size - int32_t n_batch = 8; // batch size for prompt processing - int32_t n_keep = 0; // number of tokens to keep from initial prompt (-1 for all) + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); + int32_t n_predict = 128; // new tokens to predict + int32_t repeat_last_n = 64; // last n tokens to penalize + int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions) + int32_t n_ctx = 512; // context size + int32_t n_batch = 8; // batch size for prompt processing + int32_t n_keep = 0; // number of tokens to keep from initial prompt // sampling parameters int32_t top_k = 40; @@ -33,15 +33,8 @@ struct gpt_params { std::string prompt = ""; std::string input_prefix = ""; // string to prefix user inputs with - std::string instruct_prefix = ""; // prefix user inputs with tokenized string - bool instruct_prefix_bos = false; // prepend bos token to instruct prefix - std::string instruct_suffix = ""; // suffix user inputs with tokenized string - bool instruct_suffix_bos = false; // prepend bos token to instruct suffix std::vector antiprompt; // string upon seeing which more user input is prompted - std::vector stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes) - - bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts bool memory_f16 = true; // use f16 instead of f32 for memory kv bool random_prompt = false; // do not randomize prompt if none provided @@ -58,14 +51,11 @@ struct gpt_params { bool use_mlock = false; // use mlock to keep model in memory bool mem_test = false; // compute maximum memory usage bool verbose_prompt = false; // print prompt tokens before generation - - bool clean_interface = false; // hides input prefix & suffix and displays '>' - bool multiline_mode = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows }; bool gpt_params_parse(int argc, char ** argv, gpt_params & params); -void gpt_print_usage(char * argv_0, const gpt_params & params); +void gpt_print_usage(int argc, char ** argv, const gpt_params & params); std::string gpt_random_prompt(std::mt19937 & rng); @@ -105,5 +95,3 @@ void set_console_color(console_state & con_st, console_color_t color); void win32_console_init(bool enable_color); void win32_utf8_encode(const std::wstring & wstr, std::string & str); #endif - -bool get_input_text(std::string & input_text, bool escape_newline_mode); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 68b4b28..ba153cb 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -30,8 +30,7 @@ static bool is_interacting = false; #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) void sigint_handler(int signo) { set_console_color(con_st, CONSOLE_COLOR_DEFAULT); - fflush(stdout); - fflush(stderr); + printf("\n"); // this also force flush stdout. if (signo == SIGINT) { if (!is_interacting) { is_interacting=true; @@ -90,8 +89,6 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty(); - // params.prompt = R"(// this function checks if the number n is prime //bool is_prime(int n) {)"; @@ -156,20 +153,22 @@ int main(int argc, char ** argv) { } // number of tokens to keep when resetting context - if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) { + if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) { params.n_keep = (int)embd_inp.size(); } // prefix & suffix for instruct mode - const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos); - std::string instruct_suffix = params.instruct_suffix; - if (params.rm_trailing_space_workaround) { - if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); } + const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true); + const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false); + + // in instruct mode, we inject a prefix and a suffix to each input by the user + if (params.instruct) { + params.interactive_start = true; + params.antiprompt.push_back("### Instruction:\n\n"); } - const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos); // enable interactive mode if reverse prompt or interactive start is specified - if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { + if (params.antiprompt.size() != 0 || params.interactive_start) { params.interactive = true; } @@ -211,21 +210,10 @@ int main(int argc, char ** argv) { fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str()); } } - if (params.stopprompt.size()) { - for (auto stopprompt : params.stopprompt) { - fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str()); - } - } if (!params.input_prefix.empty()) { fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str()); } - if (!params.instruct_prefix.empty()) { - fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str()); - } - if (!params.instruct_suffix.empty()) { - fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str()); - } } fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty); @@ -241,29 +229,12 @@ int main(int argc, char ** argv) { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) " - Press Ctrl+C to interject at any time.\n" #endif - ); - if (params.multiline_mode) { - fprintf(stderr, " - Press Return to return control to LLaMa.\n" -#if defined (_WIN32) - " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n"); -#else - " - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n"); -#endif - } - else { - fprintf(stderr, " - Press Return to return control to LLaMa.\n" - " - If you want to submit another line, end your input in '\\'.\n\n"); - } + " - Press Return to return control to LLaMa.\n" + " - If you want to submit another line, end your input in '\\'.\n\n"); is_interacting = params.interactive_start; } - struct Antiprompt { - bool any = false; - bool trailing_space = false; - size_t len; - bool is_stop_prompt = false; - } antiprompt; - + bool is_antiprompt = false; bool input_noecho = false; int n_past = 0; @@ -333,7 +304,7 @@ int main(int argc, char ** argv) { } // replace end of text token with newline token when in interactive mode - if (id == llama_token_eos() && params.interactive && !instruct_mode) { + if (id == llama_token_eos() && params.interactive && !params.instruct) { id = llama_token_newline.front(); if (params.antiprompt.size() != 0) { // tokenize and inject first reverse prompt @@ -379,72 +350,27 @@ int main(int argc, char ** argv) { // check if we should prompt the user for more if (params.interactive && (int) embd_inp.size() <= n_consumed) { - // check for reverse prompt or stop prompt - if (params.antiprompt.size() || params.stopprompt.size()) { + // check for reverse prompt + if (params.antiprompt.size()) { std::string last_output; for (auto id : last_n_tokens) { last_output += llama_token_to_str(ctx, id); } - antiprompt.any = false; - antiprompt.is_stop_prompt = false; + is_antiprompt = false; // Check if each of the reverse prompts appears at the end of the output. - for (std::string & prompt : params.antiprompt) { - if (params.rm_trailing_space_workaround) { - antiprompt.trailing_space = prompt.back() == ' '; - antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); - } - if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { + for (std::string & antiprompt : params.antiprompt) { + if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) { is_interacting = true; - antiprompt.any = true; + is_antiprompt = true; set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); fflush(stdout); break; } } - if (!antiprompt.any) { - for (std::string & prompt : params.stopprompt) { - if (params.rm_trailing_space_workaround) { - antiprompt.trailing_space = prompt.back() == ' '; - antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); - } - if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { - is_interacting = true; - antiprompt.any = true; - antiprompt.is_stop_prompt = true; - set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); - fflush(stdout); - break; - } - } - } } - if (n_past > 0 && is_interacting) - { - std::string buffer; - if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) { - // avoid printing again user's new line (TODO: try to revert enter press and print newline) - int i = params.instruct_prefix.front() == '\n' ? 1 : 0; - for (; i < inp_pfx.size(); i++) { - printf("%s", llama_token_to_str(ctx, inp_pfx[i])); - } - fflush(stdout); - } - if (params.rm_trailing_space_workaround) { - // add only if not stopprompt (as stopprompt could be used to pause - // assistant and then continue without input - adding back trailing - // space may mess it up.) - if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) { - // add back removed trailing space to buffer(workaround) - buffer += ' '; - if (!params.clean_interface) { - printf("%s", buffer.c_str()); - } - fflush(stdout); - } - } - + if (n_past > 0 && is_interacting) { // potentially set color to indicate we are taking user input set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); @@ -453,45 +379,49 @@ int main(int argc, char ** argv) { signal(SIGINT, sigint_handler); #endif - if (params.clean_interface) { + if (params.instruct) { printf("\n> "); } + std::string buffer; if (!params.input_prefix.empty()) { buffer += params.input_prefix; printf("%s", buffer.c_str()); } - if (!get_input_text(buffer, params.multiline_mode)) { - // input stream is bad - return 1; - } - if (!antiprompt.is_stop_prompt) { - buffer += "\n"; - } + std::string line; + bool another_line = true; + do { +#if defined(_WIN32) + std::wstring wline; + if (!std::getline(std::wcin, wline)) { + // input stream is bad or EOF received + return 0; + } + win32_utf8_encode(wline, line); +#else + if (!std::getline(std::cin, line)) { + // input stream is bad or EOF received + return 0; + } +#endif + if (line.empty() || line.back() != '\\') { + another_line = false; + } else { + line.pop_back(); // Remove the continue character + } + buffer += line + '\n'; // Append the line to the result + } while (another_line); // done taking input, reset color set_console_color(con_st, CONSOLE_COLOR_DEFAULT); - if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) { - // avoid printing again user's new line (TODO: try to revert enter press and print newline) - int i = params.instruct_suffix.front() == '\n' ? 1 : 0; - for (; i < inp_sfx.size(); i++) { - printf("%s", llama_token_to_str(ctx, inp_sfx[i])); - } - // if (remove trailing space workaround) { - // We won't add back removed trailing space here, because assistant continues here, - // and it may mess up it's output (remove trailing space workaround). - // } - fflush(stdout); - } - // Add tokens to embd only if the input buffer is non-empty // Entering a empty line lets the user pass control back if (buffer.length() > 1) { - // insert input prefix - if (!params.instruct_prefix.empty() && !antiprompt.any) { + // instruct mode: insert instruction prefix + if (params.instruct && !is_antiprompt) { n_consumed = embd_inp.size(); embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end()); } @@ -499,8 +429,8 @@ int main(int argc, char ** argv) { auto line_inp = ::llama_tokenize(ctx, buffer, false); embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end()); - // insert response suffix - if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) { + // instruct mode: insert response suffix + if (params.instruct) { embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end()); } @@ -517,7 +447,7 @@ int main(int argc, char ** argv) { // end of text token if (!embd.empty() && embd.back() == llama_token_eos()) { - if (instruct_mode) { + if (params.instruct) { is_interacting = true; } else { fprintf(stderr, " [end of text]\n"); diff --git a/prompts/alpaca.txt b/prompts/alpaca.txt new file mode 100644 index 0000000..2224bde --- /dev/null +++ b/prompts/alpaca.txt @@ -0,0 +1 @@ +Below is an instruction that describes a task. Write a response that appropriately completes the request. diff --git a/prompts/chat-with-bob.txt b/prompts/chat-with-bob.txt new file mode 100644 index 0000000..ad494d8 --- /dev/null +++ b/prompts/chat-with-bob.txt @@ -0,0 +1,7 @@ +Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. + +User: Hello, Bob. +Bob: Hello. How may I help you today? +User: Please tell me the largest city in Europe. +Bob: Sure. The largest city in Europe is Moscow, the capital of Russia. +User: \ No newline at end of file