diff --git a/CompanyConfig/Default/ChatChainConfig.json b/CompanyConfig/Default/ChatChainConfig.json index c0220536..4b273ae0 100644 --- a/CompanyConfig/Default/ChatChainConfig.json +++ b/CompanyConfig/Default/ChatChainConfig.json @@ -96,7 +96,8 @@ "clear_structure": "True", "gui_design": "True", "git_management": "False", + "web_spider": "False", "self_improve": "False", "incremental_develop": "False", "background_prompt": "ChatDev is a software company powered by multiple intelligent agents, such as chief executive officer, chief human resources officer, chief product officer, chief technology officer, etc, with a multi-agent organizational structure and the mission of 'changing the digital world through programming'." -} +} \ No newline at end of file diff --git a/CompanyConfig/Default/PhaseConfig.json b/CompanyConfig/Default/PhaseConfig.json index e44fd5ef..fb249bbc 100644 --- a/CompanyConfig/Default/PhaseConfig.json +++ b/CompanyConfig/Default/PhaseConfig.json @@ -36,6 +36,7 @@ "phase_prompt": [ "According to the new user's task and our software designs listed below: ", "Task: \"{task}\".", + "Task description: \"{description}\".", "Modality: \"{modality}\".", "Programming Language: \"{language}\"", "Ideas:\"{ideas}\"", diff --git a/camel/web_spider.py b/camel/web_spider.py new file mode 100644 index 00000000..31e1f70d --- /dev/null +++ b/camel/web_spider.py @@ -0,0 +1,89 @@ +import requests +from bs4 import BeautifulSoup +import openai +from openai import OpenAI +import wikipediaapi +import os +import time + +self_api_key = os.environ.get('OPENAI_API_KEY') +BASE_URL = os.environ.get('BASE_URL') + +if BASE_URL: + client = openai.OpenAI( + api_key=self_api_key, + base_url=BASE_URL, + ) +else: + client = openai.OpenAI( + api_key=self_api_key + ) + +def get_baidu_baike_content(keyword): + # design api by the baidubaike + url = f'https://baike.baidu.com/item/{keyword}' + # post request + response = requests.get(url) + + # Beautiful Soup part for the html content + soup = BeautifulSoup(response.content, 'html.parser') + # find the main content in the page + # main_content = soup.find('div', class_='lemma-summary') + main_content = soup.contents[-1].contents[0].contents[4].attrs['content'] + # find the target content + # content_text = main_content.get_text().strip() + return main_content + + +def get_wiki_content(keyword): + # Wikipedia API ready + wiki_wiki = wikipediaapi.Wikipedia('MyProjectName (merlin@example.com)', 'en') + #the topic content which you want to spider + search_topic = keyword + # get the page content + page_py = wiki_wiki.page(search_topic) + # check the existence of the content in the page + if page_py.exists(): + print("Page - Title:", page_py.title) + print("Page - Summary:", page_py.summary) + else: + print("Page not found.") + return page_py.summary + + + +def modal_trans(task_dsp): + try: + task_in ="'" + task_dsp + \ + "'Just give me the most important keyword about this sentence without explaining it and your answer should be only one keyword." + messages = [{"role": "user", "content": task_in}] + response = client.chat.completions.create(messages=messages, + model="gpt-3.5-turbo-16k", + temperature=0.2, + top_p=1.0, + n=1, + stream=False, + frequency_penalty=0.0, + presence_penalty=0.0, + logit_bias={}) + response_text = response.choices[0].message.content + spider_content = get_wiki_content(response_text) + # time.sleep(1) + task_in = "'" + spider_content + \ + "',Summarize this paragraph and return the key information." + messages = [{"role": "user", "content": task_in}] + response = client.chat.completions.create(messages=messages, + model="gpt-3.5-turbo-16k", + temperature=0.2, + top_p=1.0, + n=1, + stream=False, + frequency_penalty=0.0, + presence_penalty=0.0, + logit_bias={}) + result = response.choices[0].message.content + print("web spider content:", result) + except: + result = '' + print("the content is none") + return result \ No newline at end of file diff --git a/chatdev/chat_chain.py b/chatdev/chat_chain.py index b883db01..1823fac9 100644 --- a/chatdev/chat_chain.py +++ b/chatdev/chat_chain.py @@ -11,6 +11,7 @@ from camel.configs import ChatGPTConfig from camel.typing import TaskType, ModelType from chatdev.chat_env import ChatEnv, ChatEnvConfig from chatdev.statistics import get_info +from camel.web_spider import modal_trans from chatdev.utils import log_visualize, now @@ -59,6 +60,7 @@ class ChatChain: # init chatchain config and recruitments self.chain = self.config["chain"] self.recruitments = self.config["recruitments"] + self.web_spider = self.config["web_spider"] # init default max chat turn self.chat_turn_limit_default = 10 @@ -243,6 +245,8 @@ class ChatChain: self.chat_env.env_dict['task_prompt'] = self.self_task_improve(self.task_prompt_raw) else: self.chat_env.env_dict['task_prompt'] = self.task_prompt_raw + if(check_bool(self.web_spider)): + self.chat_env.env_dict['task_description'] = modal_trans(self.task_prompt_raw) def post_processing(self): """ diff --git a/chatdev/chat_env.py b/chatdev/chat_env.py index 1a75e280..b0c1756e 100644 --- a/chatdev/chat_env.py +++ b/chatdev/chat_env.py @@ -57,6 +57,7 @@ class ChatEnv: self.env_dict = { "directory": "", "task_prompt": "", + "task_description":"", "modality": "", "ideas": "", "language": "", diff --git a/chatdev/phase.py b/chatdev/phase.py index 1bc20172..2e0e0f78 100644 --- a/chatdev/phase.py +++ b/chatdev/phase.py @@ -324,6 +324,7 @@ class LanguageChoose(Phase): def update_phase_env(self, chat_env): self.phase_env.update({"task": chat_env.env_dict['task_prompt'], + "description": chat_env.env_dict['task_description'], "modality": chat_env.env_dict['modality'], "ideas": chat_env.env_dict['ideas']}) @@ -345,6 +346,7 @@ class Coding(Phase): gui = "" if not chat_env.config.gui_design \ else "The software should be equipped with graphical user interface (GUI) so that user can visually and graphically use it; so you must choose a GUI framework (e.g., in Python, you can implement GUI via tkinter, Pygame, Flexx, PyGUI, etc,)." self.phase_env.update({"task": chat_env.env_dict['task_prompt'], + "description": chat_env.env_dict['task_description'], "modality": chat_env.env_dict['modality'], "ideas": chat_env.env_dict['ideas'], "language": chat_env.env_dict['language'], @@ -366,6 +368,7 @@ class ArtDesign(Phase): def update_phase_env(self, chat_env): self.phase_env = {"task": chat_env.env_dict['task_prompt'], + "description": chat_env.env_dict['task_description'], "language": chat_env.env_dict['language'], "codes": chat_env.get_codes()} diff --git a/requirements.txt b/requirements.txt index ea5ab7db..6af7632c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ tiktoken==0.4.0 virtualenv==20.23.0 Werkzeug==2.3.6 Markdown==3.4.4 -Pillow==10.1.0 \ No newline at end of file +Pillow==10.1.0 +Wikipedia-API==0.6.0 \ No newline at end of file