information-retrieval

researcher.cpp (6056B)

---

 #include <stdexcept>
 #include <iostream>
 #include "../include/nlohmann/json.hpp"
 #include "../include/openai.hpp"
 #include "../include/researcher.hpp"
 #include <cpr/cpr.h>
 
 
 std::string Researcher::downloadSite(std::string url){
     std::cout << "Downloading " << url << std::endl;
     
     std::string command = "lynx --dump --nolist --display_charset=US-ASCII " + url;
     
     FILE* pipe = popen(command.c_str(), "r");
     std::string result;
     char buffer[4096];
     while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
         result += buffer;
     }
     
     pclose(pipe);
     
     if (result.length() > 10000) {
         result = result.substr(0, 10000) + "\n... (content truncated)";
     }
     
     return "DOWNLOADED CONTENT:\n" + result;
 }
 
 std::vector<SearchResult> Researcher::webSearch(std::string url, std::string query){
 
     std::cout << "Web searching for: " + query << std::endl;
 
     std::vector<SearchResult> returnValues;
 
     cpr::Response r = cpr::Get(
         cpr::Url{url + "/search"},
         cpr::Parameters{{"q", query}, {"format", "json"}}
     );
     // TODO: Check status code
     //std::cout << r.status_code << std::endl;
     //std::cout << r.text << std::endl;
 
     nlohmann::json currentJson = nlohmann::json::parse(r.text);
     nlohmann::json results = currentJson["results"];
 
     for (int i = 0 ; i < results.size(); ++i){
         auto currentResult = SearchResult();
         currentResult.url = results[i]["url"].get<std::string>();
         currentResult.content= results[i]["content"].get<std::string>();
         currentResult.score = results[i]["score"].get<float>();
         currentResult.title = results[i]["title"].get<std::string>();
         returnValues.push_back(currentResult);
     }
 
     std::cout << "Found " << results.size() << " search results"<< std::endl;
     return returnValues;
 }
 
 
 
 nlohmann::json Researcher::sendUserMessage(openai::OpenAI* connection , std::string model, std::string message){
 
     if(messages.size() == 0){
         throw std::runtime_error("Cannot send first message, system prompt not set.");
     }
 
     nlohmann::json userMessage = {
         {"role", "user"},
         {"content", message}
     };
     messages.push_back(userMessage);
 
     nlohmann::json messageHistory = nlohmann::json::array();
     for (const auto& msg : messages){
         messageHistory.push_back(msg);
     }
 
     nlohmann::json payload = {
         {"model", model},
         {"messages", messages},
         {"temperature", 0}
     };
 
     auto chat = connection->chat.create(
         payload
     );
 
     return chat;
 }
 
 std::string Researcher::executeTool(std::string output){
     std::string returnStr = "";
     std::vector<std::string> keywords = {"Search_web", "Download_webpage", "Done"};
 
     for(size_t i = 0; i < output.size(); ++i){
         for(size_t x = 0; x < keywords.size(); ++x){
             std::string current_word = keywords[x];
             
             if(i + current_word.size() > output.size()) continue;
             
             if(output.substr(i, current_word.size()) == current_word){
                 
                 size_t paren_start = i + current_word.size();
                 
                 if(paren_start >= output.size() || output[paren_start] != '('){
                     continue;
                 }
                 
                 size_t content_start = paren_start + 1;
                 
                 size_t content_end = content_start;
                 int paren_depth = 1;
                 
                 while(content_end < output.size() && paren_depth > 0){
                     if(output[content_end] == '(') paren_depth++;
                     else if(output[content_end] == ')') paren_depth--;
                     if(paren_depth > 0) content_end++;
                 }
                 
                 std::string inside = "";
                 if(content_end > content_start){
                     inside = output.substr(content_start, content_end - content_start);
                 }
                 
                 if(inside.size() >= 2 && inside.front() == '"' && inside.back() == '"'){
                     inside = inside.substr(1, inside.size() - 2);
                 }
                 if(inside.size() >= 4 && inside.substr(0, 2) == "\\\"" && inside.substr(inside.size() - 2) == "\\\""){
                     inside = inside.substr(2, inside.size() - 4);
                 }
                 
                 if(current_word == "Search_web"){
                     std::vector<SearchResult> results = webSearch("https://searx.laack.co", inside);
                     returnStr += "WEB SEARCH RESULTS:\n";
                     for(size_t z = 0; z < results.size(); ++z){
                         returnStr += "URL: " + results[z].url + "\n";
                         returnStr += "TITLE: " + results[z].title + "\n";
                         returnStr += "CONTENT: " + results[z].content + "\n";
                     }
                 }
                 
                 if(current_word == "Download_webpage"){
                     returnStr += downloadSite(inside);
                 }
                 
                 if(current_word == "Done"){
                     return "DONE";
                 }
                 
                 i = content_end;
             }
         }
     }
     return returnStr;
 }
 
 std::string Researcher::getDocs(){
     std::string result = "";
     // skip system prompt.
     for(int i = 1; i < messages.size(); ++i){
         result += messages[i].dump(2);
         result += "\n\n";
     }
     return result;
 }
 
 std::string Researcher::getMessageFromChat(nlohmann::json json){
     std::string result = json["choices"][0]["message"]["content"].get<std::string>();
     return result;
 }
 
 void Researcher::setSystemPrompt(std::string prompt){
     if (messages.size() == 0){
         nlohmann::json userMessage = {
             {"role", "system"},
             {"content", prompt}
         };
         messages.push_back(userMessage);
     }
     else{
         throw std::runtime_error("Cannot set system prompt after first message is sent.");
     }
 
     return;
 }