How to gather up all of the coordinate information needed for exporting a model

Question

I have been stuck for a long time on this problem, so I hope that someone can shed some light and allow me to move forwards.

Like a few other people in this forum, I want to export the coordinates of vertices in a model. My code works for some models and not others, so there must be some flag or setting that I am not taking into account, but I searched everywhere and can't see what it might be.

When I visualize my exported data in WebGL I sometimes see objects that are the correct shape and size, in the wrong location.

Note that for my purpose I want the vertices to be relative to their parent object.

You can see all of the code on GitHub, but the important part is this:

import bpy
from .logger import Logger

class FragSerializer:
    '''Converts a Blender object into a Frag DTO'''
    
    def __init__(self, context):
        self.context = context
        self.meshes = []
        self.animations = []
        self.objects = []
    
    def serialize(self, obj):
        Logger.log('Serializing animation', 0)
        self.serializeAnimations()
        Logger.log('Serialized animation', 0, 2)

        Logger.log('Serializing object ' + obj.name, 0)
        self.serializeObject(obj)
        Logger.log('Serialized object ' + obj.name, 0, 2)

        return { 
                'config': {
                    'fps': bpy.context.scene.render.fps
                },
                'animations': self.animations,
                'meshes': self.meshes,
                'objects': self.objects,
            }

    def serializeAnimations(self):
        for action in bpy.data.actions:
            Logger.log('Serializing action ' + action.name)
            groups = []
            for group in action.groups:
                channels = []
                for channel in group.channels:
                    keyframes = []
                    baseY = 0
                    for keyframe in channel.keyframe_points:
                        if len(keyframes) == 0: baseY = keyframe.co.y
                        keyframes.append({
                            "type": keyframe.type,
                            "co": self.serializeCurveCoord(keyframe.co, baseY),
                            # "amplitude": self.serializeFloat(keyframe.amplitude),
                            # "back": self.serializeFloat(keyframe.back),
                            "interpolation": keyframe.interpolation,
                            # "handle_left": self.serializeCurveCoord(keyframe.handle_left, baseY),
                            # "handle_left_type": keyframe.handle_left_type,
                            # "handle_right": self.serializeCurveCoord(keyframe.handle_right, baseY),
                            # "handle_right_type": keyframe.handle_right_type,
                        })
                    channels.append({ 
                        "data_path": channel.data_path,
                        "array_index": self.serializeAxis(channel.array_index),
                        "extrapolation": channel.extrapolation,
                        "keyframes": keyframes})
                groups.append({
                    "name": group.name,
                    "channels": channels})
            self.animations.append({ 
                "name": action.name,
                "frames": self.serializeRange(action.frame_range),
                "groups": groups })

    def serializeAxis(self, v):
        return v

    def serializePosition(self, v, baseX, baseY, baseZ):
        return [round(v.x - baseX, 4), round(v.y - baseY, 4), round(v.z - baseZ, 4)]

    def serializeColor(self, v):
        return [round(v.r, 4), round(v.g, 4), round(v.b, 4)]

    def serializeCurveCoord(self, v, baseY):
        return [round(v.x, 4), round(v.y - baseY, 4)]

    def serializeRange(self, v):
        return [round(v.x, 4), round(v.y, 4)]

    def serializeFloat(self, v):
        return round(v, 4)

    def serializeObject(self, obj):
        for child in obj.children:
            self.serializeObject(child)

        Logger.log('Serializing object ' + obj.name)
        if obj.mode == 'EDIT':
            Logger.log('Applying unsaved edits in ' + obj.name)
            obj.update_from_editmode()

        locationX = obj.location.x + obj.delta_location.x
        locationY = obj.location.y + obj.delta_location.y 
        locationZ = obj.location.z + obj.delta_location.z

        rotationX = obj.rotation_euler.x + obj.delta_rotation_euler.x
        rotationY = obj.rotation_euler.y + obj.delta_rotation_euler.y
        rotationZ = obj.rotation_euler.z + obj.delta_rotation_euler.z

        scaleX = obj.scale.x * obj.delta_scale.x
        scaleY = obj.scale.y * obj.delta_scale.y
        scaleZ = obj.scale.z * obj.delta_scale.z

        if obj.parent:
            locationX -= obj.parent.location.x + obj.parent.delta_location.x
            locationY -= obj.parent.location.y + obj.parent.delta_location.y
            locationZ -= obj.parent.location.z + obj.parent.delta_location.z

            rotationX -= obj.parent.rotation_euler.x + obj.parent.delta_rotation_euler.x
            rotationY -= obj.parent.rotation_euler.y + obj.parent.delta_rotation_euler.y
            rotationZ -= obj.parent.rotation_euler.z + obj.parent.delta_rotation_euler.z

            scaleX /= obj.parent.scale.x * obj.parent.delta_scale.x
            scaleY /= obj.parent.scale.y * obj.parent.delta_scale.y
            scaleZ /= obj.parent.scale.z * obj.parent.delta_scale.z

        serialization = { 
            'name': obj.name, 
            'location': [
                round(locationX, 4), 
                round(locationY, 4), 
                round(locationZ, 4)], 
            'rotation': [
                round(rotationX, 4), 
                round(rotationY, 4), 
                round(rotationZ, 4)], 
            'scale': [
                round(scaleX, 4), 
                round(scaleY, 4), 
                round(scaleZ, 4)], 
            'children': [ child.name for child in obj.children ]
            }

        if obj.type == 'MESH':
            # Only 1 action can be assigned to an object via animation data
            # Multiple actions are only possible via NLA tracks
            action = None
            ad = obj.animation_data
            if not ad is None:
                action = obj.animation_data.action
                if not action is None:
                    serialization["animation"] = action.name

            serialization['mesh'] = self.serializeMesh(obj, action)

        self.objects.append(serialization)

    def serializeMesh(self, obj, action):
        try:
            mesh = obj.to_mesh()
        except RuntimeError:
            return ''

        if mesh is None:
            return ''

        baseX = obj.location.x + obj.delta_location.x
        baseY = obj.location.y + obj.delta_location.y
        baseZ = obj.location.z + obj.delta_location.z

        if obj.parent:
            baseX += obj.parent.location.x + obj.parent.delta_location.x
            baseY += obj.parent.location.y + obj.parent.delta_location.y
            baseZ += obj.parent.location.z + obj.parent.delta_location.z

        if action != None:
            for group in action.groups:
                for channel in group.channels:
                    if channel.data_path == 'delta_location' or channel.data_path == 'location':
                        firstKeyframe = channel.keyframe_points[0]
                        if channel.array_index == 0: baseX += firstKeyframe.co.y
                        elif channel.array_index == 1: baseY += firstKeyframe.co.y
                        elif channel.array_index == 2: baseZ += firstKeyframe.co.y

        id = mesh.name
        mat = obj.matrix_world
        mesh.transform(mat)
        if mat.is_negative:
            mesh.flip_normals()
        mesh.calc_loop_triangles()

        vertices = mesh.vertices
        triangles = mesh.loop_triangles

        Logger.log('Mesh {} has {} triangles formed from {} verticies'.format(mesh.name, len(triangles), len(vertices)), 2)

        serialization = { 
            'id': mesh.name,
            'materials': [material.name for material in mesh.materials],
            'vertices': [self.serializePosition(v.co, baseX, baseY, baseZ) for v in vertices],
            'triangles': [{
                "vertices": [tri.vertices[0], tri.vertices[1], tri.vertices[2]], 
                "normal": self.serializePosition(tri.normal, 0, 0, 0)
                } for tri in triangles], 
        }

        obj.to_mesh_clear()

        self.meshes.append(serialization)

        return id

I have attached two screen captures below. The first is a screen grab from Blender, the second is the same model after export/import. As you can see most of the meshes are correct but some meshes are displaced and I can't see any difference between them in Blender.

Answer 1

After many hours of trying to unravel this (pretty broken) implementation, I have discovered enough about how it works to be worth sharing.

Bear in mind that I am no expert, this is just what I was able to figure out by playing around in Blender and running the following script:

import bpy

indentStr = '    '

def printObj(obj, indent):
    print()
    print(indentStr * indent, obj.name, ' ', obj.type)
    print(indentStr * indent, 'location: ', obj.location, ' ', obj.delta_location)
    print(indentStr * indent, 'euler: ', obj.rotation_euler, ' ', obj.delta_rotation_euler)
    print(indentStr * indent, 'quaternion: ', obj.rotation_quaternion, ' ', obj.delta_rotation_quaternion)
    print(indentStr * indent, 'scale: ', obj.scale, ' ', obj.delta_scale)
    print(indentStr * indent, 'dimensions: ', obj.dimensions)
    print(indentStr * indent, 'matrix_basis: ', obj.matrix_basis.decompose())
    print(indentStr * indent, 'matrix_local: ', obj.matrix_local.decompose())
    print(indentStr * indent, 'matrix_parent_inverse: ', obj.matrix_parent_inverse.decompose())
    print(indentStr * indent, 'matrix_world: ', obj.matrix_world.decompose())
    
    if obj.type == 'MESH':
        mesh = obj.to_mesh()
        mesh.calc_loop_triangles()
        
    for child in obj.children: printObj(child, indent + 1)

obj = bpy.context.object
while obj.parent: obj = obj.parent
printObj(obj, 0)

You can run this script in the scripting window in Blender, and it will print out the object hierarchy with all of the information relevant to the positioning of vertices on the screen.

This is what I learned from playing around with this script:

1. matrix_local seems to be just the matrix combination of location, rotation_euler, rotation_quaternion and scale plus their delta_xx versions.
2. Taking a vertex in the mesh and multiplying it by matrix_world returns the position where that vertex will be drawn regardless of how deeply nested the object is within the parent/child hierarchy.
3. Taking a vertex in the mesh and multiplying it by matrix_local and by the parent's matrix_world results in the same thing as just multiplying by matrix_world.

Where things get a bit messed up, is when you change the parent of an object. All of the places where my models are broken are places where the parent/child structure was changed at some point. In this case the matrix_basis and matrix_parent_inverse also come into play.

The matrix_basis seems to be the same as matrix_local for objects that don't have a parent or always had the same parent. For these objects the matrix_parent_inverse does not change anything (no translation, rotation or scale).

For objects that had their parent changed at some point, the "Transform" and "Delta Transform" in the "Object Properties" window no longer tell you the position of the object in the scene because the matrix_basis no longer matches matrix_local and matrix_parent_inverse defines a transform that does something.

Nobody would design it like this on purpose, so I can only assume this is the result of adding new features whilst maintaining backwards compatibility, resulting in a very convoluted and excessively complicated implementation.

My best guess for matrix_parent_inverse is that it is the inverse of the matrix_world for the previous parent. this would suggest that to get from local coordinate space to world coordinate space we should multiply vertices by the matrix_parent_inverse before multiplying by the current parent's matrix_world.

Through my various experiments I have been unable so far to figure out what the matrix_basis is for or how/when to use it. It is always the same as matrix_local until you re-parent an object, so it's probably related to matrix_parent_inverse somehow, but the documentation is non-existent.

The other complication that you need to be aware of is that once you add an animation, the first keyframe of the animation contributes to the position of the object in the scene, so you need to find the first keyframe and add and translations, rotations or scaling present in that keyframe channel to the vertex coordinates to make them correct.

When you step through the keyframes of an animation, all of these values change except for matrix_parent_inverse. If you want to export the model, and the animation is part way through, then you will need to reverse all of the prior animation steps to find the correct initial state of the model for export.

This is my updated code, which is working for all of my current models:

import bpy
import mathutils
from .logger import Logger

class FragSerializer:
    '''Converts a Blender object into a Frag DTO'''
    
    def __init__(self, context):
        self.context = context
        self.meshes = []
        self.animations = []
        self.objects = []
    
    def serialize(self, obj):
        Logger.log('Serializing animation', 0)
        self.serializeAnimations()
        Logger.log('Serialized animation', 0, 2)

        Logger.log('Serializing object ' + obj.name, 0)
        self.serializeObject(obj)
        Logger.log('Serialized object ' + obj.name, 0, 2)

        return { 
                'config': {
                    'fps': bpy.context.scene.render.fps
                },
                'animations': self.animations,
                'meshes': self.meshes,
                'objects': self.objects,
            }

    def serializeAnimations(self):
        for action in bpy.data.actions:
            Logger.log('Serializing action ' + action.name)
            groups = []
            for group in action.groups:
                channels = []
                for channel in group.channels:
                    keyframes = []
                    baseY = 0
                    for keyframe in channel.keyframe_points:
                        if len(keyframes) == 0: baseY = keyframe.co.y
                        keyframes.append({
                            "type": keyframe.type,
                            "co": self.serializeCurveCoord(keyframe.co, baseY),
                            # "amplitude": self.serializeFloat(keyframe.amplitude),
                            # "back": self.serializeFloat(keyframe.back),
                            "interpolation": keyframe.interpolation,
                            # "handle_left": self.serializeCurveCoord(keyframe.handle_left, baseY),
                            # "handle_left_type": keyframe.handle_left_type,
                            # "handle_right": self.serializeCurveCoord(keyframe.handle_right, baseY),
                            # "handle_right_type": keyframe.handle_right_type,
                        })
                    channels.append({ 
                        "data_path": channel.data_path,
                        "array_index": self.serializeAxis(channel.array_index),
                        "extrapolation": channel.extrapolation,
                        "keyframes": keyframes})
                groups.append({
                    "name": group.name,
                    "channels": channels})
            self.animations.append({ 
                "name": action.name,
                "frames": self.serializeRange(action.frame_range),
                "groups": groups })

    def serializeAxis(self, v):
        return v

    def serializePosition(self, v, baseX, baseY, baseZ):
        return [round(v.x - baseX, 4), round(v.y - baseY, 4), round(v.z - baseZ, 4)]

    def serializeColor(self, v):
        return [round(v.r, 4), round(v.g, 4), round(v.b, 4)]

    def serializeCurveCoord(self, v, baseY):
        return [round(v.x, 4), round(v.y - baseY, 4)]

    def serializeRange(self, v):
        return [round(v.x, 4), round(v.y, 4)]

    def serializeFloat(self, v):
        return round(v, 4)

    def serializeObject(self, obj):
        for child in obj.children:
            self.serializeObject(child)

        Logger.log('Serializing object ' + obj.name)
        if obj.mode == 'EDIT':
            Logger.log('Applying unsaved edits in ' + obj.name, 2)
            obj.update_from_editmode()

        location, rotation, scale = (obj.matrix_local).decompose()
        Logger.log('Loc, rot, scale ' + str(location) + str(rotation) + str(scale), 2)

        serialization = { 
            'name': obj.name, 
            'location': [
                round(location.x, 4), 
                round(location.y, 4), 
                round(location.z, 4)], 
            'rotation': [
                round(rotation.x, 4), 
                round(rotation.y, 4), 
                round(rotation.z, 4)], 
            'scale': [
                round(scale.x, 4), 
                round(scale.y, 4), 
                round(scale.z, 4)], 
            'children': [ child.name for child in obj.children ]
            }

        if obj.type == 'MESH':
            # Only 1 action can be assigned to an object via animation data
            # Multiple actions are only possible via NLA tracks
            action = None
            ad = obj.animation_data
            if not ad is None:
                action = obj.animation_data.action
                if not action is None:
                    serialization["animation"] = action.name

            serialization['mesh'] = self.serializeMesh(obj, action)

        self.objects.append(serialization)

    def serializeMesh(self, obj, action):
        try:
            mesh = obj.to_mesh()
        except RuntimeError:
            return ''

        if mesh is None:
            return ''

        base = mathutils.Vector()

        if action != None:
            for group in action.groups:
                for channel in group.channels:
                    if channel.data_path == 'delta_location' or channel.data_path == 'location':
                        firstKeyframe = channel.keyframe_points[0]
                        if channel.array_index == 0: base.x += firstKeyframe.co.y
                        elif channel.array_index == 1: base.y += firstKeyframe.co.y
                        elif channel.array_index == 2: base.z += firstKeyframe.co.y

        id = mesh.name
        mesh.calc_loop_triangles()

        vertices = mesh.vertices
        triangles = mesh.loop_triangles

        Logger.log('Mesh {} has {} triangles formed from {} verticies'.format(mesh.name, len(triangles), len(vertices)), 2)

        serialization = { 
            'id': mesh.name,
            'materials': [material.name for material in mesh.materials],
            'vertices': [self.serializePosition(v.co, base.x, base.y, base.z) for v in vertices],
            'triangles': [{
                "smooth": tri.use_smooth,
                "vertices": [tri.vertices[0], tri.vertices[1], tri.vertices[2]], 
                "normal": self.serializePosition(tri.normal, 0, 0, 0)
                } for tri in triangles], 
        }

        obj.to_mesh_clear()

        self.meshes.append(serialization)

        return id
```