the results of world_to_camera_view( ) are not correct

Question

I want to get the 2D coordinates of some vertices and bones' head on my 3D mesh model. Here is my code related to this process.

 def get_joint_loacation(self, joint_name):
        armature = bpy.data.objects.get(self.armature_name)
        bone = armature.pose.bones.get(joint_name)
        assert armature and bone, 'No armature/bone found for %s' % joint_name
        global_location = armature.matrix_world @ bone.head
        return global_location

 def get_keyponits_coords(self):
        mesh = bpy.data.objects.get(self.mesh_name)
        vertices = mesh.data.vertices

        for key, value in self.keypoints_name.items():
            if type(value)==str:
                self.keypoints[key] = self.get_joint_loacation(value)
            elif type(value)==int:
                self.keypoints[key] = mesh.matrix_world @ vertices[value].co
            else:
                raise ValueError(f'Unknown keypoint type: {type(value)}') 

 def get_2D_labels(self, camera, extent):
        labels={}
        res_x = self.scene.render.resolution_x
        res_y = self.scene.render.resolution_y
        for key, value in self.keypoints.items():
            coords_2d = world_to_camera_view(self.scene, camera, value)
            x_2d = coords_2d.x*res_x
            y_2d = coords_2d.y*res_y
            labels[key] = (x_2d, y_2d)
            print(key,coords_2d)


        # create bbox
        if labels['left_first_toe_tip']!=None and labels['left_heel']!=None:
            toe = labels['left_first_toe_tip']
            heel = labels['left_heel']
            left_top = (min(toe[0],heel[0])-extent,max(toe[1],heel[1])+extent)
            right_bottom = (max(toe[0],heel[0])+extent,min(toe[1],heel[1])-extent)
            labels['left_bbox'] = [left_top, right_bottom]
        else:
            labels['left_bbox'] = None

        if labels['right_first_toe_tip']!=None and labels['right_heel']!=None:
            toe = labels['right_first_toe_tip']
            heel = labels['right_heel']
            left_top = (min(toe[0],heel[0])-extent,max(toe[1],heel[1])+extent)
            right_bottom = (max(toe[0],heel[0])+extent,min(toe[1],heel[1])-extent)
            labels['right_bbox'] = [left_top, right_bottom]
        else:
            labels['right_bbox'] = None

        return labels

 def create(self, idx, radius=10, num_views=4, lens=60, extent=10):
        # set camera properties and select
        bpy.ops.object.select_all(action='DESELECT')
        cam_ob = bpy.data.objects['Camera']
        cam_ob.select_set(True)
        cam_ob.animation_data_clear()
        cam_ob.data.lens = lens
        cam_ob.data.clip_start = 0.1
        cam_ob.data.sensor_width = 32

        # render from sample points locations
        for bone in ['left_foot','right_foot']:
            target = self.get_joint_loacation(bone)
            sample_points = self.sample_points_around(target, radius, num_views)
            for i, point in enumerate(sample_points):
                if bone=='left_foot':
                    image_name = "sub_{:05d}_left_{:02}".format(idx,i)
                if bone=='right_foot':
                    image_name = "sub_{:05d}_right_{:02}".format(idx,i)

                # set camera location and rotation
                direction = (point - target).normalized()
                cam_ob.location = point
                cam_ob.rotation_euler = direction.to_track_quat('Z', 'Y').to_euler()
                # cam_ob.rotation_euler.rotate_axis('Z', math.radians(90))

                # get 2D labels
                label_path = self.save_path.joinpath('labels')
                label_path.mkdir(parents=True, exist_ok=True)
                label_file = label_path.joinpath(image_name+'.json')
                with label_file.open('w') as f:
                    json.dump(self.get_2D_labels(cam_ob, extent=extent),f)
                print("Saved labels to: ", label_file)

                # set random bg
                self.set_random_bg()

                # render
                bpy.context.scene.render.filepath = str(self.save_path.joinpath('rgb',image_name+'.png'))
                bpy.ops.render.render(write_still=True)

I gathered these functions in a class just to keep main() short and clean. By running create(), I want to rotate the camera around my model and capture the scene from 8 points on a surface whose center is a bone.head's location named left_foot`right_foot. And then output the picture from these views and the 2D coordinates of 3D keypoints in the picture by using world_to_camera_view()`.

By now, the output pictures are fine, but the 2D coordinates of keypoints are not. I print the output of world_to_camera_view(), found the result was out camera view field, cause the y is negative.

But when I view from the camera, the keypoints are in the field.

And in order to see the 2D points look like, I simply turn the y to positive and plot by pyplot.

As you can see the structure of keypoints seems all right, they stand for toe, foot_mid and ankle location. But they seems like upside down or rotate by an error angle.

I read all the questions about world_to_camera_view(). I checked the matrix_word. My picture is fine, so I think the camera was also correct, and I was confused. Where I did wrong?

Answer 1

I found the solution. It's not because of world_to_camera_view(), or camera, or anything about the computation. I changed the mode to Edit Mode to pose the armature and forgot to change back. So the location I input to world_to_camera_view() are under Edit Mode, which didn't posed at all. That's why I see the target in render result but the result of world_to_camera_view() are out of camera's view. Always make sure you change the mode to Pose Mode before you get the location of bones.