MineDojo · jiqingsong · Jan 18, 2024 · Jan 18, 2024 · Jan 18, 2024 · Jan 18, 2024
diff --git a/crawl.py b/crawl.py
@@ -0,0 +1,50 @@
+import time
+import json
+import urllib
+import requests
+import parsel
+
+url = 'https://minecraft.wiki/w/Block'
+header = {"User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122  Safari / 537.36"}
+base_url = 'https://minecraft.wiki'
+
+response = requests.get(url=url, headers=header)
+response.encoding = "utf-8"
+html_data = response.text
+
+html = response.text
+response.close()
+html = html.encode('gbk','ignore')
+html = html.decode("gbk")
+with open('content.html', 'w', encoding='utf-8') as f:
+    f.write(html_data)
+
+rag_dict = {}
+selector = parsel.Selector(html)
+result_list = selector.xpath('//div[@class="div-col columns column-width" and @style="-moz-column-width: 19em; -webkit-column-width: 19em; column-width: 19em;"]/ul/li')
+
+for result in result_list:
+    content = result.xpath('./a')[-1]
+    # print(content)
+    title = content.xpath('./@title').get()
+    href = content.xpath('./@href').get()
+    name = href.split('/')[-1]
+    sub_url = base_url + href
+    print(name, sub_url)
+    s = requests.session()
+    s.keep_alive = False
+    response = requests.get(url=sub_url, headers=header)
+    html = response.text
+    response.close()
+    # with open('content.html', 'w') as f:
+    #     f.write(html)
+    selector = parsel.Selector(html)
+    description = selector.xpath('//meta[@name="description"]')
+    description = description.xpath('./@content').get()
+    # print(name, description)
+    # break
+    rag_dict[name.lower()] = description
+    time.sleep(0.1)
+
+with open('rag_dict.json', 'w', encoding='utf-8') as f:
+    f.write(json.dumps(rag_dict, ensure_ascii=False, indent=4))
diff --git a/voyager/agents/action.py b/voyager/agents/action.py
@@ -1,6 +1,6 @@
 import re
 import time
-
+import json
 import voyager.utils as U
 from javascript import require
 from langchain.chat_models import ChatOpenAI
@@ -36,6 +36,8 @@ def __init__(
             temperature=temperature,
             request_timeout=request_timout,
         )
+        with open("voyager/agents/rag_dict.json", 'r', encoding='utf-8') as f:
+            self.rag_dict = json.load(f)
 
     def update_chest_memory(self, chests):
         for position, chest in chests.items():
@@ -88,6 +90,7 @@ def render_system_message(self, skills=[]):
                 "useChest",
                 "mineflayer",
             ]
+        # Behavioral cloning
         programs = "\n\n".join(load_control_primitives_context(base_skills) + skills)
         response_format = load_prompt("action_response_format")
         system_message_prompt = SystemMessagePromptTemplate.from_template(
@@ -152,8 +155,136 @@ def render_human_message(
 
         observation += f"Time: {time_of_day}\n\n"
 
+        #TODO Retrieval Augmented Generation: Add block information
+        # print('================================================================================')
+        # print("voxels: ", type(voxels))
+        # print(voxels)
+        if voxels:
+            observation += f"Nearby blocks: {', '.join(voxels)}\n"
+            for voxel in voxels:
+                if voxel in self.rag_dict.keys():
+                    observation += '\t' + voxel + ': ' + self.rag_dict[voxel] + "\n"
+            observation += '\n'
+        else:
+            observation += f"Nearby blocks: None\n\n"
+
+        if entities:
+            nearby_entities = [
+                k for k, v in sorted(entities.items(), key=lambda x: x[1])
+            ]
+            observation += f"Nearby entities (nearest to farthest): {', '.join(nearby_entities)}\n\n"
+        else:
+            observation += f"Nearby entities (nearest to farthest): None\n\n"
+
+        observation += f"Health: {health:.1f}/20\n\n"
+
+        observation += f"Hunger: {hunger:.1f}/20\n\n"
+
+        observation += f"Position: x={position['x']:.1f}, y={position['y']:.1f}, z={position['z']:.1f}\n\n"
+
+        observation += f"Equipment: {equipment}\n\n"
+
+        if inventory:
+            observation += f"Inventory ({inventory_used}/36): {inventory}\n\n"
+        else:
+            observation += f"Inventory ({inventory_used}/36): Empty\n\n"
+
+        if not (
+            task == "Place and deposit useless items into a chest"
+            or task.startswith("Deposit useless items into the chest at")
+        ):
+            observation += self.render_chest_observation()
+
+        observation += f"Task: {task}\n\n"
+
+        if context:
+            observation += f"Context: {context}\n\n"
+        else:
+            observation += f"Context: None\n\n"
+
+        if critique:
+            observation += f"Critique: {critique}\n\n"
+        else:
+            observation += f"Critique: None\n\n"
+
+        return HumanMessage(content=observation)
+
+    def render_system_planer_message(self):
+        system_template = load_prompt("planner_template")
+        # FIXME: Hardcoded control_primitives
+        response_format = load_prompt("planner_response_format")
+        system_message_prompt = SystemMessagePromptTemplate.from_template(
+            system_template
+        )
+        system_message = system_message_prompt.format(
+            response_format=response_format
+        )
+        assert isinstance(system_message, SystemMessage)
+        return system_message
+
+    def render_human_planer_message(
+        self, *, events, code="", task="", context="", critique=""
+    ):
+        chat_messages = []
+        error_messages = []
+        # FIXME: damage_messages is not used
+        damage_messages = []
+        assert events[-1][0] == "observe", "Last event must be observe"
+        for i, (event_type, event) in enumerate(events):
+            if event_type == "onChat":
+                chat_messages.append(event["onChat"])
+            elif event_type == "onError":
+                error_messages.append(event["onError"])
+            elif event_type == "onDamage":
+                damage_messages.append(event["onDamage"])
+            elif event_type == "observe":
+                biome = event["status"]["biome"]
+                time_of_day = event["status"]["timeOfDay"]
+                voxels = event["voxels"]
+                entities = event["status"]["entities"]
+                health = event["status"]["health"]
+                hunger = event["status"]["food"]
+                position = event["status"]["position"]
+                equipment = event["status"]["equipment"]
+                inventory_used = event["status"]["inventoryUsed"]
+                inventory = event["inventory"]
+                assert i == len(events) - 1, "observe must be the last event"
+
+        observation = ""
+
+        if code:
+            observation += f"Code from the last round:\n{code}\n\n"
+        else:
+            observation += f"Code from the last round: No code in the first round\n\n"
+
+        if self.execution_error:
+            if error_messages:
+                error = "\n".join(error_messages)
+                observation += f"Execution error:\n{error}\n\n"
+            else:
+                observation += f"Execution error: No error\n\n"
+
+        if self.chat_log:
+            if chat_messages:
+                chat_log = "\n".join(chat_messages)
+                observation += f"Chat log: {chat_log}\n\n"
+            else:
+                observation += f"Chat log: None\n\n"
+
+        observation += f"Biome: {biome}\n\n"
+
+        observation += f"Time: {time_of_day}\n\n"
+
+        #TODO Retrieval Augmented Generation: Add block information
+        # print('================================================================================')
+        # print("voxels: ", type(voxels))
+        # print(voxels)
         if voxels:
-            observation += f"Nearby blocks: {', '.join(voxels)}\n\n"
+            observation += f"Nearby blocks: {', '.join(voxels)}\n"
+            for voxel in voxels:
+                if voxel in self.rag_dict.keys():
+                    observation += '\t' + voxel + ': ' + self.rag_dict[voxel] + "\n"
+            observation += '\n'
         else:
             observation += f"Nearby blocks: None\n\n"