From cc6dbf513197b925036b7e49829a5322f93ed77f Mon Sep 17 00:00:00 2001 From: Jiahao-Li <111221887+JiahaoLi2003@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:07:13 +0800 Subject: [PATCH 1/4] Update ECL with ECE --- ecl/ece.py | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 ecl/ece.py diff --git a/ecl/ece.py b/ecl/ece.py new file mode 100644 index 00000000..d2e6a09e --- /dev/null +++ b/ecl/ece.py @@ -0,0 +1,146 @@ + +import os +import json +import re +import numpy as np +import argparse +point = 0.95 +eliminate_threshold = 0.95 + + +def retrieve_eliminate(directory_Path,directory_Memory,eliminated_directory): + experiences_use = [] + content = [] + content1 = [] + experiences_total = [] + usetime_total = [] + exp_dict = {} + eliminated_exp = [] + + directories = [os.path.join(directory_Path, d) for d in os.listdir(directory_Path) if os.path.isdir(os.path.join(directory_Path, d))] + for subdir in directories: + directory = subdir + logdir = [filename for filename in os.listdir(directory) if filename.endswith(".log")] + logdir = os.path.join(directory, logdir[0]) + content1 = open(logdir, "r", encoding='UTF-8').read() + + pattern1 = re.compile(r'the source code MIDs is (.*?),', re.S) + experiences_sourceMIDs = re.findall(pattern1, content1) + pattern2 = re.compile(r'the target code MIDs is (.*?)\n',re.S) + experiences_targetMIDs = re.findall(pattern2, content1) + pattern3 = re.compile(r'And the (.*?) similarity is',re.S) + experiences_type = re.findall(pattern3,content1) + for i in range(0,len(experiences_sourceMIDs)): + sourceMID = experiences_sourceMIDs[i] + targetMID = experiences_targetMIDs[i] + type = experiences_type[i] + experiences_use.append((sourceMID,targetMID,type)) + + with open(directory_Memory) as file: + content1 = json.load(file) + new_content = [] + for memorypiece in content1: + experiences = memorypiece.get("experiences") + if experiences != None: + experiences_total.extend(experiences) + for experience in experiences: + experience["use_time"] = 0 + for experience in experiences_use: + for experience_t in experiences_total: + if experience[0] == experience_t["sourceMID"] and experience[1] == experience_t["targetMID"]: + experience_t["use_time"] += 1 + for i,experience_t in enumerate(experiences_total): + usetime_total.append(experience_t["use_time"]) + exp_dict[i] = experience_t["use_time"] + file.close() + + usetime_sort = sorted(usetime_total)[::-1] + total = np.sum(usetime_sort) + for i in range(len(usetime_sort)): + if np.sum(usetime_sort[:i])/total >= point: + # print("α：",i) + alpha= i + break + index=0 + for k in sorted(exp_dict,key=exp_dict.__getitem__,reverse=True): + if index <= alpha: + eliminated_exp.append(experiences_total[k]) + index += 1 + else: + break + + for memorypiece in content1: + experiences = memorypiece.get("experiences") + retrieve_eliminated_experienceList = [] + if experiences != None: + for experience in experiences: + if experience in eliminated_exp: + retrieve_eliminated_experienceList.append(experience) + + memorypiece["experiences"] = retrieve_eliminated_experienceList + new_content.append(memorypiece) + + with open(eliminated_directory, 'w') as file: + json.dump(new_content, file) + + +# Quality score gain Elimination +def gain_eliminate(directory_NewMemory,eliminated_directory): + content2 = [] + with open(directory_NewMemory) as file: + content2 = json.load(file) + new_content2 = [] + for memorypiece in content2: + experiences = memorypiece.get("experiences") + gain_eliminated_experienceList = [] + + if experiences != None: + # print("origin:", len(experiences)) + for experience in experiences: + valueGain = experience.get("valueGain") + # print(valueGain) + if valueGain >= eliminate_threshold: + gain_eliminated_experienceList.append(experience) + # print(len(experiences)) + memorypiece["experiences"] = gain_eliminated_experienceList + new_content2.append(memorypiece) + else: + new_content2.append(memorypiece) + file.close() + + with open(eliminated_directory, 'r') as file: + new_content = json.load(file) + + new_content = new_content + new_content2 + + with open(eliminated_directory, 'w') as file: + json.dump(new_content, file) + + + +def recount_experience(eliminated_directory): + with open(eliminated_directory, 'r') as file: + content = json.load(file) + + with open(eliminated_directory, 'w') as file: + i = 0 + for memorypiece in content: + memorypiece["total"] = i + i += 1 + json.dump(content, file) + +def main(): + parser = argparse.ArgumentParser(description="Process memory with some directories.") + parser.add_argument("directory_Path", type = str, help="The directory of software") + parser.add_argument("directory_Memory", type=str, help="The directory of MemoryCards") + parser.add_argument("directory_NewMemory", type=str, help="The directory of NewMemoryCards") + parser.add_argument("eliminated_directory", type= str, help="The directory for output") + + + args = parser.parse_args() + retrieve_eliminate(args.directory_Path,args.directory_Memory,args.eliminated_directory) + gain_eliminate(args.directory_NewMemory,args.eliminated_directory) + recount_experience(args.eliminated_directory) + +if __name__ == "__main__": + main() From cf57e9d6490816a2b803b2425b30335df81c2cfe Mon Sep 17 00:00:00 2001 From: Jiahao-Li <111221887+JiahaoLi2003@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:46:46 +0800 Subject: [PATCH 2/4] Update ece.py --- ecl/ece.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ecl/ece.py b/ecl/ece.py index d2e6a09e..e7c615bd 100644 --- a/ecl/ece.py +++ b/ecl/ece.py @@ -8,7 +8,7 @@ point = 0.95 eliminate_threshold = 0.95 -def retrieve_eliminate(directory_Path,directory_Memory,eliminated_directory): +def retrieve_eliminate(Path_directory,UsedMemory_directory,Evolved_directory): experiences_use = [] content = [] content1 = [] @@ -17,7 +17,7 @@ def retrieve_eliminate(directory_Path,directory_Memory,eliminated_directory): exp_dict = {} eliminated_exp = [] - directories = [os.path.join(directory_Path, d) for d in os.listdir(directory_Path) if os.path.isdir(os.path.join(directory_Path, d))] + directories = [os.path.join(Path_directory, d) for d in os.listdir(Path_directory) if os.path.isdir(os.path.join(Path_directory, d))] for subdir in directories: directory = subdir logdir = [filename for filename in os.listdir(directory) if filename.endswith(".log")] @@ -36,7 +36,7 @@ def retrieve_eliminate(directory_Path,directory_Memory,eliminated_directory): type = experiences_type[i] experiences_use.append((sourceMID,targetMID,type)) - with open(directory_Memory) as file: + with open(UsedMemory_directory) as file: content1 = json.load(file) new_content = [] for memorypiece in content1: @@ -80,14 +80,14 @@ def retrieve_eliminate(directory_Path,directory_Memory,eliminated_directory): memorypiece["experiences"] = retrieve_eliminated_experienceList new_content.append(memorypiece) - with open(eliminated_directory, 'w') as file: + with open(Evolved_directory, 'w') as file: json.dump(new_content, file) # Quality score gain Elimination -def gain_eliminate(directory_NewMemory,eliminated_directory): +def gain_eliminate(NewMemory_directory,Evolved_directory): content2 = [] - with open(directory_NewMemory) as file: + with open(NewMemory_directory) as file: content2 = json.load(file) new_content2 = [] for memorypiece in content2: @@ -108,21 +108,21 @@ def gain_eliminate(directory_NewMemory,eliminated_directory): new_content2.append(memorypiece) file.close() - with open(eliminated_directory, 'r') as file: + with open(Evolved_directory, 'r') as file: new_content = json.load(file) new_content = new_content + new_content2 - with open(eliminated_directory, 'w') as file: + with open(Evolved_directory, 'w') as file: json.dump(new_content, file) -def recount_experience(eliminated_directory): - with open(eliminated_directory, 'r') as file: +def recount_experience(Evolved_directory): + with open(Evolved_directory, 'r') as file: content = json.load(file) - with open(eliminated_directory, 'w') as file: + with open(Evolved_directory, 'w') as file: i = 0 for memorypiece in content: memorypiece["total"] = i @@ -131,16 +131,16 @@ def recount_experience(eliminated_directory): def main(): parser = argparse.ArgumentParser(description="Process memory with some directories.") - parser.add_argument("directory_Path", type = str, help="The directory of software") - parser.add_argument("directory_Memory", type=str, help="The directory of MemoryCards") - parser.add_argument("directory_NewMemory", type=str, help="The directory of NewMemoryCards") - parser.add_argument("eliminated_directory", type= str, help="The directory for output") + parser.add_argument("Path_directory", type = str, help="The directory of software") + parser.add_argument("UsedMemory_directory", type=str, help="The directory of MemoryCards") + parser.add_argument("NewMemory_directory", type=str, help="The directory of NewMemoryCards") + parser.add_argument("Evolved_directory", type= str, help="The directory for output") args = parser.parse_args() - retrieve_eliminate(args.directory_Path,args.directory_Memory,args.eliminated_directory) - gain_eliminate(args.directory_NewMemory,args.eliminated_directory) - recount_experience(args.eliminated_directory) + retrieve_eliminate(args.Path_directory,args.UsedMemory_directory,args.Evolved_directory) + gain_eliminate(args.NewMemory_directory,args.Evolved_directory) + recount_experience(args.Evolved_directory) if __name__ == "__main__": main() From f5fa8167c1761d005f8dc27177595f1052a16203 Mon Sep 17 00:00:00 2001 From: Jiahao-Li <111221887+JiahaoLi2003@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:48:31 +0800 Subject: [PATCH 3/4] Update wiki.md --- wiki.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/wiki.md b/wiki.md index f67a84fc..ac9afd1f 100644 --- a/wiki.md +++ b/wiki.md @@ -149,7 +149,7 @@ then start building a software by ``python3 run.py`` and go to [Visualizer Websi After this process, the experiences have been extracted from the production of software and added to the agents' experience pool in `ecl/memory/MemoryCards.json`. \ **For example:** - It you want to memorize only one software, you can use: + Iff you want to memorize only one software, you can use: ```bash python3 ecl/ecl.py "" ``` @@ -187,6 +187,25 @@ After this process, the experiences have been extracted from the production of s Detailed descriptions and experiment results about this **Experiential Co-Learning** Module lies in our preprint paper at https://arxiv.org/abs/2312.17025. +## Experiential Co-Evolving Guide +- **Using Co-Evolving**: Use the following command to initiate the evolving of experiences, which uses the `ecl/ece.py` to eliminate `ecl/memory/UsedMemory.json` and `ecl/memory/NewMemory.json`. Then it combines the two parts of experiences to form a new experience pool in `ecl/memory/Evolved_directory.json`. + + ```bash + python3 ecl/ece.py "" "" "" "" + ``` + ``: The path to the directory of software , generated with the memory `UsedMemory_directory`. \ + ``: The path to the directory of UsedMemory, which was used to generate the software in `Path_directory`. \ + ``: The path to the directory NewMemory, which acquires from the software in `Path_directory` using `ecl/ecl.py`. \ + ``: The path to a directory where you want to store the evolved memory. + \ + **For example:** + ```bash + python3 ecl/ece.py "WareHouse" "ecl/memory/UsedMemory.json" "ecl/memory/NewMemory.json" "ecl/memory/MemoryCards_Evolved.json" + ``` +> **Notice:** The software directory and memory directory must correspond. The software in the "" is generated using "", and the "" is acquired from the software in the "". That's because when we calculate the frequency distribution of the experience, we need to ensure that the software corresponds to the experiences, which to eliminate certain experiences to obtain a subset with relatively high retrieval probability. + +Detailed descriptions and experiment results about this Experiential Co-Evolving Module lies in our preprint paper at https://arxiv.org/abs/2405.04219. + ## Customization - You can customize your company in three kinds of granularity: From 997d84cfb7ad422e60028c959e54e8882620ea51 Mon Sep 17 00:00:00 2001 From: Jiahao-Li <111221887+JiahaoLi2003@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:20:52 +0800 Subject: [PATCH 4/4] Update wiki.md --- wiki.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki.md b/wiki.md index ac9afd1f..688f7f18 100644 --- a/wiki.md +++ b/wiki.md @@ -149,7 +149,7 @@ then start building a software by ``python3 run.py`` and go to [Visualizer Websi After this process, the experiences have been extracted from the production of software and added to the agents' experience pool in `ecl/memory/MemoryCards.json`. \ **For example:** - Iff you want to memorize only one software, you can use: + If you want to memorize only one software, you can use: ```bash python3 ecl/ecl.py "" ```