ChatDev/chatdev/competition.py

import os
import glob
import time
import random
import tiktoken
from openai import OpenAI
import re
import signal
import subprocess
from chatdev.codes import Codes
from chatdev.utils import log_macnet
from chatdev.test_unfinished_function import FunctionTest


def competition_filter(dir_program):
    def getFilesFromType(sourceDir, filetype):
        files = []
        for root, directories, filenames in os.walk(sourceDir):
            for filename in filenames:
                if filename.endswith(filetype):
                    files.append(os.path.join(root, filename))
        return files

    def get_code(directory):
        def _format_code(code):
            code = "\n".join([line for line in code.split("\n") if len(line.strip()) > 0])
            return code

        # Read all .py files
        codebooks = {}
        filepaths = getFilesFromType(directory, ".py")
        for filepath in filepaths:
            filename = os.path.basename(filepath)
            codebooks[filename] = _format_code(open(filepath, "r", encoding="utf-8").read())

        # Format Codes
        code = ""
        for filename in codebooks.keys():
            code += "{}\n```Python\n{}\n```\n\n".format(filename, codebooks[filename])

        if len(code) == 0:
            code = "# None"

        return code.strip()

    def get_completeness(directory):
        vn = get_code(directory)
        lines = vn.split("\n")

        lines = [line for line in lines if
                 "password" not in line.lower() and "passenger" not in line.lower() and "passed" not in line.lower() and "passes" not in line.lower()]
        lines = [line for line in lines if "pass" in line.lower() or "todo" in line.lower()]
        if len(lines) > 0:
            return 0.0
        return 1.0

    def get_executability(directory):
        def findFile(directory, target):
            main_py_path = None
            for subroot, _, filenames in os.walk(directory):
                for filename in filenames:
                    if filename == target:
                        main_py_path = os.path.join(subroot, filename)
            return main_py_path

        def exist_bugs(directory):
            success_info = "The software run successfully without errors."
            try:
                command = "cd \"{}\"; ls -l; python3 main.py;".format(directory)
                process = subprocess.Popen(command, shell=True, preexec_fn=os.setsid, stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
                time.sleep(1)  # to 3 once stuck

                error_type = ""
                return_code = process.returncode
                # Check if the software is still running
                if process.poll() is None:
                    timeout = 10
                    try:
                        os.killpg(os.getpgid(process.pid), signal.SIGTERM)
                        process.wait(timeout=timeout)
                    except subprocess.TimeoutExpired:
                        os.killpg(os.getpgid(process.pid), signal.SIGKILL)
                        process.wait()
                if return_code == 0:
                    return False, success_info, error_type
                else:
                    error_output = process.stderr.read().decode('utf-8')
                    try:
                        error_pattern = r'\w+Error:'
                        error_matches = re.findall(error_pattern, error_output)
                        error_type = error_matches[0].replace(":", "")
                    except:
                        pass
                    if error_output:
                        if "Traceback".lower() in error_output.lower():
                            errs = error_output.replace(directory + "/", "")
                            return True, errs, error_type
                    else:
                        return False, success_info, error_type

            except subprocess.CalledProcessError as e:
                return True, f"Error: {e}", "subprocess.CalledProcessError"
            except Exception as ex:
                return True, f"An error occurred: {ex}", "OtherException"

            return False, success_info, error_type

        main_py_path = findFile(directory, "main.py")
        pass_flag, error_type = True, ""
        if main_py_path is not None:
            main_py_path = os.path.dirname(main_py_path)
            bug_flag, info, error_type = exist_bugs(main_py_path)
            pass_flag = not bug_flag
        else:
            pass_flag, error_type = False, "NoMain"

        if error_type == "":
            error_type = info.replace("\n", "\\n")

        if pass_flag:
            return 1.0
        return 0.0

    def get_code_line(directory):
        total_lines = 0
        for root, dirs, files in os.walk(directory):
            for file_name in files:
                if file_name.endswith(".py") and file_name != 'test_file_will_delete.py':
                    file_path = os.path.join(root, file_name)

                    with open(file_path, 'r', encoding='utf-8') as file:
                        lines = file.readlines()
                        total_lines += len(lines)

        return total_lines

    completeness_dict = {}
    executability_dict = {}
    code_line_dict = {}
    competition_dict = {}
    completeness = get_completeness(dir_program)
    executability = get_executability(dir_program)
    # code_line = get_code_line(dir_program)
    function_test = FunctionTest(dir_program)
    need_complete = function_test.extract_function_name()
    # completeness_dict[content] = completeness
    # executability_dict[content] = executability
    # code_line_dict[content] = code_line
    need_complete_number = 0
    for function in need_complete.values():
        need_complete_number += len(function)
    rate_number = (executability * 10 + completeness) * 10000 - need_complete_number
    del function_test

    return rate_number