论文信息

1

python learn.py --file examples/a_stack_of_seven_plates.glb --desc "a stack of seven plates"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

def load_glb(file_path: str) -> tuple[gltf.GLTF2, list[Obj]]:
    '''
    Load a glb file.

    Args:
        file_path: string, the path to the glb file
    
    Returns:
        glb_file: GLTF2, the glTF 2.0 file
        objs_in_file: list, the objects in the file
    '''

    # TODO: Make less assumptions about the structure of the glTF file

    glb_file = gltf.GLTF2().load(file_path)
    objs_in_file: list[Obj] = []
    
    # Load the main scene and the scene's world nodes
    all_nodes = glb_file.nodes
    main_scene = glb_file.scenes[glb_file.scene]
    world_node_ids = main_scene.nodes
    
    # Load the objects in the world nodes
    for world_node_id in world_node_ids:
        main_node_ids = all_nodes[world_node_id].children

        # Load the objects in the main nodes
        # Each main node is a separate object
        for main_node_id in main_node_ids:
            main_node = all_nodes[main_node_id]

            # ----- Matrix -----
            # Note: This logic is not general!
            # Assumes that the mesh_root_node has a matrix for the mesh
            # And the main_node has a matrix for placing the mesh in the world
            if main_node.matrix:
                main_node_matrix = np.array(main_node.matrix).reshape(4, 4, order="F")
            else:
                translation = main_node.translation if main_node.translation else [0, 0, 0]
                rotation = main_node.rotation if main_node.rotation else [1, 0, 0, 0]
                scale = main_node.scale if main_node.scale else [1, 1, 1]
                translation_matrix = trimesh.transformations.translation_matrix(translation)
                rotation_matrix = trimesh.transformations.quaternion_matrix(rotation)
                scale_matrix = np.diag(scale + [1])
                main_node_matrix = translation_matrix @ rotation_matrix @ scale_matrix

            if len(main_node.children) > 0:
                mesh_root_node_matrix = all_nodes[main_node.children[0]].matrix
                if mesh_root_node_matrix is not None:
                    mesh_root_node_matrix = np.array(mesh_root_node_matrix).reshape(4, 4, order="F")
            else:
                mesh_root_node_matrix = None

            # ----- Mesh -----
            main_node_mesh = _load_all_meshes(glb_file, main_node)
            
            # Apply the mesh_root_node_matrix if it exists
            if mesh_root_node_matrix is not None:
                main_node_mesh.apply_transform(mesh_root_node_matrix)

            # ----- Bounding box -----
            # Compute the oriented bounding box of the mesh
            centroid = main_node_mesh.bounding_box_oriented.centroid + main_node_matrix[:3, 3]
            half_size = main_node_mesh.bounding_box_oriented.extents / 2
            main_node_bounding_box = BoundingBox(centroid, half_size, main_node_matrix[:3, :3])
            
            # ----- Obj -----
            # TODO: Allow the label to be provided separately
            label = main_node.extras["semantics"]["label"] if "semantics" in main_node.extras else "unknown"

            obj = Obj(label, main_node_bounding_box, main_node_mesh, main_node_matrix)
            objs_in_file.append(obj)

    return glb_file, objs_in_file

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

def from_arrangement(cls, arrangement: Arrangement) -> Program:
    '''
    Create a program from an arrangement.

    Args:
        arrangement: Arrangement, an arrangement of objects

    Returns:
        program: Program, the program of the arrangement
    '''

    code = []
    code.append("objs = []")

    for i, obj in enumerate(arrangement.objs):
        label = obj.label
        centroid = obj.bounding_box.centroid.tolist()
        half_size = obj.bounding_box.half_size.tolist()
        coord_axes = obj.bounding_box.coord_axes

        # The object is initially aligned with the canonical coordinate system, so we need to rotate it to the correct orientation
        # For user convenience, the rotation operation is decomposed into three rotations around the x, y, and z axes
        # Here, we need to extract the three rotation angles needed to rotate the object to the correct orientation
        # In particular, we expect most objects only has a single rotation around the up axis (y axis), so we choose a rotation order of YXZ
        rotation_matrix = coord_axes
        y_angle = np.arctan2(rotation_matrix[0, 2], rotation_matrix[2, 2])
        x_angle = np.arcsin(-rotation_matrix[1, 2])
        z_angle = np.arctan2(rotation_matrix[1, 0], rotation_matrix[1, 1])
        
        code.append(f"obj_{i+1}_half_size = {half_size}")
        code.append(f"obj_{i+1}_centroid = {centroid}")
        code.append(f"obj_{i+1} = create('{label}', obj_{i+1}_half_size)")
        code.append(f"move(obj_{i+1}, obj_{i+1}_centroid[0], obj_{i+1}_centroid[1], obj_{i+1}_centroid[2])")

        for axis, angle in [("y", y_angle), ("x", x_angle), ("z", z_angle)]:
            if abs(angle) > 1e-5:
                code.append(f"rotate(obj_{i+1}, '{axis}', {np.rad2deg(angle).round(1)})")
        
        code.append(f"objs.append(obj_{i+1})")
    
    program = Program(code, arrangement.description)
    return program

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

Naive program:
objs = []
obj_1_half_size = [0.08909, 0.0143, 0.08853]
obj_1_centroid = [0.0, 0.0, 0.0]
obj_1 = create('plate', obj_1_half_size)
move(obj_1, obj_1_centroid[0], obj_1_centroid[1], obj_1_centroid[2])
objs.append(obj_1)
obj_2_half_size = [0.08909, 0.0143, 0.08853]
obj_2_centroid = [0.0, -0.00757, 0.0]
obj_2 = create('plate', obj_2_half_size)
move(obj_2, obj_2_centroid[0], obj_2_centroid[1], obj_2_centroid[2])
objs.append(obj_2)
obj_3_half_size = [0.08909, 0.0143, 0.08853]
obj_3_centroid = [0.0, -0.01514, 0.0]
obj_3 = create('plate', obj_3_half_size)
move(obj_3, obj_3_centroid[0], obj_3_centroid[1], obj_3_centroid[2])
objs.append(obj_3)
obj_4_half_size = [0.08909, 0.0143, 0.08853]
obj_4_centroid = [0.0, -0.02272, -0.0]
obj_4 = create('plate', obj_4_half_size)
move(obj_4, obj_4_centroid[0], obj_4_centroid[1], obj_4_centroid[2])
objs.append(obj_4)
obj_5_half_size = [0.08909, 0.0143, 0.08853]
obj_5_centroid = [0.0, -0.03029, -0.0]
obj_5 = create('plate', obj_5_half_size)
move(obj_5, obj_5_centroid[0], obj_5_centroid[1], obj_5_centroid[2])
objs.append(obj_5)
obj_6_half_size = [0.08909, 0.0143, 0.08853]
obj_6_centroid = [0.0, -0.03786, -0.0]
obj_6 = create('plate', obj_6_half_size)
move(obj_6, obj_6_centroid[0], obj_6_centroid[1], obj_6_centroid[2])
objs.append(obj_6)
obj_7_half_size = [0.08909, 0.0143, 0.08853]
obj_7_centroid = [0.0, -0.04544, 0.0]
obj_7 = create('plate', obj_7_half_size)
move(obj_7, obj_7_centroid[0], obj_7_centroid[1], obj_7_centroid[2])
objs.append(obj_7)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

def make_high_level_observations(llm_session: gpt.Session, naive_program: Program) -> None:
    '''
    Prompt the LLM to make high level observations of the naive program.

    Args:
        llm_session: Session, the LLM session
        naive_program: Program, the naive program to observe
    
    Returns:
        None
    '''

    llm_session.send("optimize_highlevel_count", {"program": naive_program.code_string, "description": naive_program.description})
    llm_session.send("optimize_highlevel_general_pattern", {"description": naive_program.description})
    llm_session.send("optimize_highlevel_xyz_pattern")
    llm_session.send("optimize_highlevel_xyz_displacements")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def classify_motif_type(llm_session: gpt.Session, description: str) -> str:
    '''
    Prompt the LLM to classify the motif type of the description.

    Args:
        llm_session: Session, the LLM session
        description: string, the description of the arrangement
    
    Returns:
        motif_type: string, the classified motif type
    '''

    with open("motif_types.yaml", "r") as f:
        motif_types = yaml.safe_load(f)["types"].keys()

    # ----- Validation function for this task -----
    def classify_validation(response: str) -> tuple[bool, str, int]:
        motif_type = response.strip().lower()
        valid = motif_type in motif_types or (motif_type.startswith("letter_") and motif_type[-1] in "abcdefghijklmnopqrstuvwxyz")
        error_message = f"The motif type '{motif_type}' is invalid. Valid motif types are: {motif_types}" if not valid else ""
        return valid, error_message, -1
    # ----- End of validation function -----

    motif_type = llm_session.send_with_validation("classify", {"description": description}, classify_validation)
    return motif_type

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

types:
  stack: multiple objects of the same type are placed orderly on top of each other
  pile: objects are placed on top of each other but not in an orderly manner (when in doubt between stack and pile, choose pile if the description contains the word "pile" explicitly)
  row: objects are placed next to each other in a row
  grid: objects are placed orderly in a grid, like a chessboard
  left_of: one object placed to the left of another object
  in_front_of: one object placed in front of another object
  on_top: one object placed on top of another object
  surround: objects are placed around a central object in a circular manner
  wall_vertical_column: objects are placed in a column from top to bottom vertically on a wall
  wall_horizontal_row: objects are placed in a row from left to right vertically on a wall
  wall_grid: objects are placed in a grid orderly and vertically on a wall (when in doubt between grid and wall_grid, choose wall_grid if the objects are placed on a wall)
  letter: objects are placed to form a letter of the alphabet
  rectangular_perimeter: objects are placed around the perimeter of a rectangular shape, facing inward

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

def optimize_motif_program(llm_session: gpt.Session, naive_program: Program, example_arrangement: Arrangement) -> Program:
    '''
    Prompt the LLM to optimize the naive program to create a motif program.

    Args:
        llm_session: Session, the LLM session
        naive_program: Program, the naive program to optimize
        example_arrangement: Arrangement, the example arrangement
    
    Returns:
        optimized_program: Program, the optimized motif program
    '''
    
    # ----- Validation function for this task -----
    def optimize_validation(response: str) -> tuple[bool, str, int]:
        program = gpt.extract_program(response, naive_program.description)

        validations = [
            validator.validate_syntax,
            validator.validate_naive_listing,
            validator.validate_num_objects,
            validator.validate_centroids,
            validator.validate_bounding_boxes,
        ]

        arguments = [
            [program],
            [program],
            [program, example_arrangement.objs],
            [program, example_arrangement.objs],
            [program, example_arrangement.objs],
        ]

        for i, (validation, argument) in enumerate(zip(validations, arguments)):
            valid, error_message = validation(*argument)
            if not valid:
                return valid, error_message, i
        
        return True, "", -1
    # ----- End of validation function -----

    optimize_response = llm_session.send_with_validation("optimize_lowlevel", None, optimize_validation)
    optimized_program = gpt.extract_program(optimize_response, naive_program.description)
    return optimized_program

1
2
3
4
5
6

objs = []

for i in range(7):
    plate = create('plate', [0.08909, 0.0143, 0.08853])
    move(plate, 0.0, i * -0.00757, 0.0)
    objs.append(plate)  # 仍然存储对象引用在objs列表中

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

def observe_commonalities_and_differences(llm_session: gpt.Session, motif_type: str) -> None:
    '''
    Prompt the LLM to observe the high level commonalities and differences in programs of the same motif type.

    Args:
        llm_session: Session, the LLM session
        motif_type: string, the motif type
    
    Returns:
        None
    '''

    # Get all programs of the same motif type
    all_programs = ""
    if library.length(motif_type) > 0:
        loaded_programs = library.load(motif_type)
        for i, program in enumerate(loaded_programs):
            all_programs += f"Program {i + 1}. '{program.description}':\n{program.code_string}\n\n"
        print(f"Loaded {len(loaded_programs)} programs of motif type: {motif_type}\n")
    else:
        raise RuntimeError(f"No programs in library for motif type: {motif_type}")

    llm_session.send("generalize_high_level_commonalities", {"num_programs": str(len(loaded_programs)),
                                                             "motif_type": motif_type,
                                                             "all_programs": all_programs})
    llm_session.send("generalize_high_level_differences")
    llm_session.send("generalize_high_level_motif_reason", {"motif_type": motif_type})

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

### Commonalities Across Programs:
1. **Consistent Use of the `create` Function**: Each program creates objects using the same type (e.g., 'plate') and size (e.g., `[0.08909, 0.0143, 0.08853]`).
2. **Modification of Object Positioning**: In all programs, object positioning is manipulated through the `move` function, typically modifying the coordinates along the primary axis of the motif (y-axis for stack, x-axis or z-axis for row/grid, and random offsets for pile).
3. **Consistency in Object Type and Size**: In each program variation, the same object type and size are used, maintaining a degree of consistency.
4. **Order and Spacing**: In most programs, objects are spaced with consistent intervals along the primary axis, although the spacing may vary (e.g., stack with a fixed offset, row with different spacing).

### Key Differences:
- The key differences between programs revolve around how the objects are positioned: 
   - A **stack** involves vertical alignment with consistent spacing.
   - A **row** involves horizontal alignment along one axis.
   - A **pile** involves random or unstructured placement, disrupting the uniformity.
   - A **grid** involves a two-dimensional arrangement, often across both x and y axes.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

def prepare_meta_program_info(llm_session: gpt.Session, motif_type: str) -> None:
    '''
    Prompt the LLM to prepare information for writing the meta-program.

    Args:
        llm_session: Session, the LLM session
        motif_type: string, the motif type
    
    Returns:
        None
    '''

    llm_session.send("generalize_low_level_arguments", {"motif_type": motif_type})
    llm_session.send("generalize_low_level_structure", {"motif_type": motif_type})

def write_meta_program(llm_session: gpt.Session, motif_type: str, refine_comments: bool = True) -> Program:
    '''
    Prompt the LLM to write the meta-program.

    Args:
        llm_session: Session, the LLM session
        motif_type: string, the motif type
        refine_comments: bool, whether to refine the comments of the meta-program
    
    Returns:
        meta_program: program, the written meta-program
    '''

    # Get all programs of the same motif type
    if library.length(motif_type) > 0:
        loaded_programs = library.load(motif_type)
    else:
        raise RuntimeError(f"No programs in library for motif type: {motif_type}")

    # Get the previous meta-program if available
    if library.length(motif_type, is_meta=True) > 0:
        past_meta_program = library.load(motif_type, is_meta=True)[0].code_string
    else:
        past_meta_program = "# NO PAST META-PROGRAM AVAILABLE"

    # ----- Validation function for this task -----
    def generalize_validation(response: str) -> tuple[bool, str, int]:
        meta_program = gpt.extract_program(response, motif_type)

        batch_recreate_response = llm_session.send("generalize_low_level_batch_recreate")
        try:
            recreate_calls: dict[str, str] = json.loads(gpt.extract_json(batch_recreate_response))
            recreate_calls = list(recreate_calls.values())
        except json.JSONDecodeError as e:
            return False, f"Failed to decode the json response: {e}", -1

        valid, error_message = validator.validate_meta_program(meta_program, recreate_calls, loaded_programs)

        return valid, error_message, 0 if not valid else -1
    # ----- End of validation function -----

    generalize_response = llm_session.send_with_validation("generalize_low_level", 
                                                           {"motif_type": motif_type, 
                                                            "past_meta_program": past_meta_program}, 
                                                            generalize_validation)
    meta_program = gpt.extract_program(generalize_response, motif_type)

    # Refine the documentation of the meta-program
    if refine_comments:
        # ----- Validation function for this task -----
        def refine_comments_validation(response: str) -> tuple[bool, str, int]:
            meta_program = gpt.extract_program(response, motif_type)
            valid, error_message = validator.validate_syntax(meta_program, require_objs=False)
            return valid, error_message, -1
        # ----- End of validation function -----

        refine_response = llm_session.send_with_validation("generalize_refine_comments", 
                                                        {"motif_type": motif_type}, 
                                                        refine_comments_validation)
        meta_program = gpt.extract_program(refine_response, motif_type)
    
    return meta_program

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

def create_stack(num_objects, obj_type, obj_size, starting_position, vertical_spacing, rotation_angle=0, axis_alignment='y'):
    """
    Creates a stack of objects based on the specified parameters. The objects are stacked along the given axis 
    with consistent spacing between each object.

    Parameters:
    - num_objects (int): The number of objects to be created and stacked. For example, 7 for a stack of seven objects.
    - obj_type (str): The type of object to be created (e.g., 'plate', 'cube'). Defines the kind of object to stack.
    - obj_size (list of floats): A list containing the half-size dimensions of the object in the x, y, and z directions.
      Example: [0.08909, 0.0143, 0.08853] for a plate. This defines the size of each object in the stack.
    - starting_position (list of floats): The initial [x, y, z] position of the first object. This sets the position of the 
      bottom-most object in the stack.
    - vertical_spacing (float): The vertical displacement (in meters) between the centers of two consecutive objects in the stack.
      For example, `-0.00757` meters.
    - rotation_angle (float, optional): The rotation angle (in degrees) for each object around its local axis. Default is 0 (no rotation).
      This can be used if you need to rotate the objects around their axes.
    - axis_alignment (str, optional): The axis along which the objects will be stacked. Options are 'x', 'y', 'z'. Default is 'y'.
      For example, use 'y' for vertical stacking, 'x' for horizontal stacking along the x-axis, and 'z' for stacking along the z-axis.

    Returns:
    - list: A list of all created objects that have been placed in the stack. The list contains the objects in the order they were stacked.

    Example call:
    create_stack(7, 'plate', [0.08909, 0.0143, 0.08853], [0.0, 0.0, 0.0], -0.00757)
    """
    
    objs = []  # Initialize an empty list to store the created objects

    # Loop to create and position each object in the stack
    for i in range(num_objects):
        # Create an object of the specified type with the given size
        obj = create(obj_type, obj_size)
        
        # Calculate the position based on the axis alignment and vertical spacing
        if axis_alignment == 'y':
            # For stacking along the y-axis (default)
            move(obj, starting_position[0], starting_position[1] + i * vertical_spacing, starting_position[2])
        elif axis_alignment == 'x':
            # For stacking along the x-axis
            move(obj, starting_position[0] + i * vertical_spacing, starting_position[1], starting_position[2])
        elif axis_alignment == 'z':
            # For stacking along the z-axis
            move(obj, starting_position[0], starting_position[1], starting_position[2] + i * vertical_spacing)

        # Apply rotation if provided
        if rotation_angle != 0:
            # Apply rotation around the x-axis by default
            rotate(obj, 'x', rotation_angle)  # This could be modified to rotate around y or z if needed

        # Append the created object to the list of objects
        objs.append(obj)

    return objs  # Return the list of objects that have been created and stacked