I want to get the 2D coordinates of some vertices and bones' head on my 3D mesh model. Here is my code related to this process.

 def get_joint_loacation(self, joint_name):
        armature = bpy.data.objects.get(self.armature_name)
        bone = armature.pose.bones.get(joint_name)
        assert armature and bone, 'No armature/bone found for %s' % joint_name
        global_location = armature.matrix_world @ bone.head
        return global_location

 def get_keyponits_coords(self):
        mesh = bpy.data.objects.get(self.mesh_name)
        vertices = mesh.data.vertices

        for key, value in self.keypoints_name.items():
            if type(value)==str:
                self.keypoints[key] = self.get_joint_loacation(value)
            elif type(value)==int:
                self.keypoints[key] = mesh.matrix_world @ vertices[value].co
                raise ValueError(f'Unknown keypoint type: {type(value)}') 

 def get_2D_labels(self, camera, extent):
        res_x = self.scene.render.resolution_x
        res_y = self.scene.render.resolution_y
        for key, value in self.keypoints.items():
            coords_2d = world_to_camera_view(self.scene, camera, value)
            x_2d = coords_2d.x*res_x
            y_2d = coords_2d.y*res_y
            labels[key] = (x_2d, y_2d)

        # create bbox
        if labels['left_first_toe_tip']!=None and labels['left_heel']!=None:
            toe = labels['left_first_toe_tip']
            heel = labels['left_heel']
            left_top = (min(toe[0],heel[0])-extent,max(toe[1],heel[1])+extent)
            right_bottom = (max(toe[0],heel[0])+extent,min(toe[1],heel[1])-extent)
            labels['left_bbox'] = [left_top, right_bottom]
            labels['left_bbox'] = None

        if labels['right_first_toe_tip']!=None and labels['right_heel']!=None:
            toe = labels['right_first_toe_tip']
            heel = labels['right_heel']
            left_top = (min(toe[0],heel[0])-extent,max(toe[1],heel[1])+extent)
            right_bottom = (max(toe[0],heel[0])+extent,min(toe[1],heel[1])-extent)
            labels['right_bbox'] = [left_top, right_bottom]
            labels['right_bbox'] = None

        return labels

 def create(self, idx, radius=10, num_views=4, lens=60, extent=10):
        # set camera properties and select
        cam_ob = bpy.data.objects['Camera']
        cam_ob.data.lens = lens
        cam_ob.data.clip_start = 0.1
        cam_ob.data.sensor_width = 32

        # render from sample points locations
        for bone in ['left_foot','right_foot']:
            target = self.get_joint_loacation(bone)
            sample_points = self.sample_points_around(target, radius, num_views)
            for i, point in enumerate(sample_points):
                if bone=='left_foot':
                    image_name = "sub_{:05d}_left_{:02}".format(idx,i)
                if bone=='right_foot':
                    image_name = "sub_{:05d}_right_{:02}".format(idx,i)

                # set camera location and rotation
                direction = (point - target).normalized()
                cam_ob.location = point
                cam_ob.rotation_euler = direction.to_track_quat('Z', 'Y').to_euler()
                # cam_ob.rotation_euler.rotate_axis('Z', math.radians(90))

                # get 2D labels
                label_path = self.save_path.joinpath('labels')
                label_path.mkdir(parents=True, exist_ok=True)
                label_file = label_path.joinpath(image_name+'.json')
                with label_file.open('w') as f:
                    json.dump(self.get_2D_labels(cam_ob, extent=extent),f)
                print("Saved labels to: ", label_file)

                # set random bg

                # render
                bpy.context.scene.render.filepath = str(self.save_path.joinpath('rgb',image_name+'.png'))

I gathered these functions in a class just to keep main() short and clean. By running create(), I want to rotate the camera around my model and capture the scene from 8 points on a surface whose center is a bone.head's location named left_foot`right_foot. And then output the picture from these views and the 2D coordinates of 3D keypoints in the picture by using world_to_camera_view()`.

By now, the output pictures are fine, but the 2D coordinates of keypoints are not. I print the output of world_to_camera_view(), found the result was out camera view field, cause the y is negative.

enter image description here

But when I view from the camera, the keypoints are in the field.

enter image description here

And in order to see the 2D points look like, I simply turn the y to positive and plot by pyplot.

enter image description here

As you can see the structure of keypoints seems all right, they stand for toe, foot_mid and ankle location. But they seems like upside down or rotate by an error angle.

I read all the questions about world_to_camera_view(). I checked the matrix_word. My picture is fine, so I think the camera was also correct, and I was confused. Where I did wrong?


1 Answer 1


I found the solution. It's not because of world_to_camera_view(), or camera, or anything about the computation. I changed the mode to Edit Mode to pose the armature and forgot to change back. So the location I input to world_to_camera_view() are under Edit Mode, which didn't posed at all. That's why I see the target in render result but the result of world_to_camera_view() are out of camera's view. Always make sure you change the mode to Pose Mode before you get the location of bones.


You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .