Edit: does not work in 2.8
I propose another approach which is not vertex colors based, mainly because vertex colors influence is limited by the inner geometry of the mesh.
Instead, we can use some kind of "cut mask", which allow to do the following:

It can be used also with more complex shapes, as soon as a boolean intersect (same as the operation in edit mode) can be applied:


bl_info = {
"name": "Generate a mask on a texture using boolean operation",
"author": "lemon",
"version": (1, 0),
"blender": (2, 78, 0),
"location": "3D View > Scene properties > Cut masks",
"description": "",
"warning": "",
"wiki_url": "",
"category": "Mesh"}
import bpy, bmesh, bgl
from mathutils import Color
def Find( l, name ):
index = l.find( name )
if index == -1:
return None
return l[index]
def GetSelection():
return bpy.context.active_object, bpy.context.selected_objects
def RestoreSelection( active, selected ):
active.select = True
for o in selected: o.select = True
bpy.context.scene.objects.active = active
def BGLInit( imageWidth, imageHeight ):
bgl.glDisable( bgl.GL_SCISSOR_TEST )
bgl.glViewport( 0, 0, imageWidth, imageHeight )
bgl.glMatrixMode( bgl.GL_PROJECTION )
bgl.glLoadIdentity()
bgl.gluOrtho2D( 0, imageWidth, 0, imageHeight )
bgl.glEnable( bgl.GL_TEXTURE_2D )
def BGLToDefault():
bgl.glEnable( bgl.GL_SCISSOR_TEST )
bgl.glLineWidth( 1 )
bgl.glDisable( bgl.GL_BLEND )
bgl.glColor4f( 0.0, 0.0, 0.0, 1.0 )
def PaintMask( cutObjects, colors, defaultColor, uvMapName, imageName, borderSize ):
image = Find( bpy.data.images, imageName )
if image is None:
return { 'INCORRECT' }
imageWidth = image.size[0]
imageHeight = image.size[1]
BGLInit( imageWidth, imageHeight )
try:
buffer = bgl.Buffer( bgl.GL_FLOAT, imageWidth * imageHeight * 4 )
bgl.glClearColor( *defaultColor, 0 )
bgl.glClear( bgl.GL_COLOR_BUFFER_BIT | bgl.GL_DEPTH_BUFFER_BIT )
bgl.glLineWidth( 2 * borderSize )
for obj, color in zip( cutObjects, colors ):
uvMap = Find( obj.data.uv_layers, uvMapName )
if uvMap:
bgl.glColor3f( *color )
for polygon in obj.data.polygons:
bgl.glBegin( bgl.GL_POLYGON )
for x, y in [uvMap.data[vIndex].uv for vIndex in polygon.loop_indices]:
bgl.glVertex2f( x * imageWidth, y * imageHeight )
bgl.glEnd()
if borderSize > 0:
bgl.glBegin( bgl.GL_LINE_LOOP )
for x, y in [uvMap.data[vIndex].uv for vIndex in polygon.loop_indices]:
bgl.glVertex2f( x * imageWidth, y * imageHeight )
bgl.glEnd()
bgl.glFinish()
bgl.glReadPixels( 0, 0, imageWidth, imageHeight, bgl.GL_RGBA, bgl.GL_FLOAT, buffer )
image.pixels = buffer[:]
del buffer
finally:
BGLToDefault()
return {'DONE'}
def MakeCopy( obj ):
newObj = obj.copy()
newObj.data = obj.data.copy()
bpy.context.scene.objects.link( newObj )
return newObj
def SelectVertices( obj, mode ):
bpy.ops.object.mode_set( mode = 'OBJECT' )
bpy.context.scene.objects.active = obj
bpy.ops.object.mode_set( mode = 'EDIT' )
bpy.ops.mesh.select_all( action = mode )
def TriangulateMesh( obj ):
bpy.ops.object.mode_set( mode = 'OBJECT' )
bm = bmesh.new()
bm.from_mesh( obj.data )
bmesh.ops.triangulate( bm, faces=bm.faces[:] )
bm.to_mesh( obj.data )
bm.free()
del bm
def MakeCut( obj, cutter ):
bpy.context.scene.objects.active = obj
bpy.ops.object.mode_set( mode = 'OBJECT' )
bpy.ops.object.select_all(action='DESELECT')
SelectVertices( cutter, 'SELECT' )
SelectVertices( obj, 'DESELECT' )
bpy.ops.object.mode_set( mode = 'OBJECT' )
cutter.select = True
obj.select = True
bpy.context.scene.objects.active = obj
bpy.ops.object.join()
try:
bpy.ops.object.mode_set( mode = 'EDIT' )
bpy.ops.mesh.intersect_boolean( operation = 'INTERSECT' )
TriangulateMesh( obj )
bpy.ops.object.mode_set( mode = 'OBJECT' )
return obj
except:
DeleteObjects( [obj, cutter] )
return None
def MakeCuts( obj, cutters ):
newObjects = [MakeCopy( o ) for o in [obj]*len(cutters)]
newCutters = [MakeCopy( o ) for o in cutters]
return [MakeCut( o, cutter ) for o, cutter in zip(newObjects, newCutters)]
def DeleteObjects( objects ):
try:
bpy.ops.object.mode_set( mode = 'OBJECT' )
for o in [o for o in objects if o]:
o.select = True
bpy.context.scene.objects.active = o
bpy.ops.object.delete()
except:
pass
def CreateMasks( obj, selectedObjects, colors, defaultColor, uvMapName, imageName, borderSize ):
cutObjects = MakeCuts( obj, selectedObjects )
PaintMask( cutObjects, colors, defaultColor, uvMapName, imageName, borderSize )
DeleteObjects( cutObjects )
def GetTargetFromScene( scn ):
obj = None
try:
if scn.cutMaskUseActive:
obj = scn.objects.active
else:
obj = scn.objects[scn.cutMaskObject]
except:
pass
return obj
class CutMaskApply( bpy.types.Operator ):
bl_idname = "cutmask.apply"
bl_label = ""
bl_options = {'REGISTER'}
@classmethod
def poll(self, context):
scn = context.scene
if not context.mode == 'OBJECT':
#scn.cutMaskError = "Must be in object mode"
return False
obj = GetTargetFromScene( scn )
if not obj or obj.type != 'MESH' or scn.cutMaskObjectUVMap not in obj.data.uv_layers:
#CutMaskApply(context).report({'INFO'}, "No mesh target object")
return False
if scn.cutMaskObjectUVMap not in obj.data.uv_layers:
#scn.cutMaskError = "Uv map undefined in target"
return False
if scn.cutMaskImage not in bpy.data.images:
#scn.cutMaskError = "Image mask undefined"
return False
cutterNames = [elt.name for elt in scn.cutMasks]
if obj.name in cutterNames:
#scn.cutMaskError = "Target object cannot cut itself"
return False
for name in cutterNames:
if name not in scn.objects:
#scn.cutMaskError = name + " is undefined"
return False
elif scn.objects[name].type != 'MESH':
#scn.cutMaskError = name + " is not a mesh"
return False
elif scn.objects[name].data.uv_layers:
#scn.cutMaskError = "Warning: " + name + " has a UV layer"
pass
return True
def invoke(self, context, event):
wasActive, wereSelected = GetSelection()
scn = context.scene
objName = scn.cutMaskObject
distance = scn.cutMaskDistance
uvMapName = scn.cutMaskObjectUVMap
imageName = scn.cutMaskImage
cutterNames = [elt.name for elt in scn.cutMasks]
cutterColors = [elt.color for elt in scn.cutMasks]
defaultColor = scn.cutMaskDefaultColor
try:
obj = GetTargetFromScene( scn )
cutters = [scn.objects[c] for c in cutterNames]
if obj.type == 'MESH' and all( o.type == 'MESH' for o in cutters ):
CreateMasks( obj, cutters, cutterColors, defaultColor, uvMapName, imageName, distance )
# except:
# print( "error level 1" )
finally:
RestoreSelection( wasActive, wereSelected )
return { 'FINISHED' }
class CutMaskListActions( bpy.types.Operator ):
bl_idname = "cutmask.list_action"
bl_label = ""
action = bpy.props.EnumProperty( items = ( ('UP', "Up", ""), ('DOWN', "Down", ""), ('REMOVE', "Remove", ""), ('ADD', "Add", ""), ) )
def invoke(self, context, event):
scn = context.scene
idx = scn.cutMaskIndex
if self.action == 'DOWN' and idx < len(scn.cutMasks) - 1:
scn.cutMasks.move( idx, idx + 1 )
scn.cutMaskIndex += 1
elif self.action == 'UP' and idx > 0:
scn.cutMasks.move( idx, idx - 1 )
scn.cutMaskIndex -= 1
elif self.action == 'REMOVE':
scn.cutMasks.remove(scn.cutMaskIndex)
scn.cutMaskIndex -= 1
elif self.action == 'ADD':
for o in [o for o in bpy.context.selected_objects if o.name not in [cm.name for cm in scn.cutMasks]]:
item = scn.cutMasks.add()
item.name = o.name
scn.cutMaskIndex = len(scn.cutMasks) - 1
return {"FINISHED"}
class CutMaskItems( bpy.types.UIList ):
def draw_item(self, context, layout, data, item, icon, active_data, active_propname, index):
split = layout.split(0.6)
split.prop( item, "name", text="", emboss=False, translate=False )
split.prop( item, "color", text="" )
def invoke(self, context, event):
pass
class CutMaskPanel( bpy.types.Panel ):
bl_idname = 'OBJECT_PT_my_panel'
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_label = "Cut masks"
def draw(self, context):
layout = self.layout
scn = bpy.context.scene
row = layout.row()
row.template_list( "CutMaskItems", "", scn, "cutMasks", scn, "cutMaskIndex", rows=3 )
col = row.column( align = True )
col.operator( "cutmask.list_action", icon='ZOOMIN', text="" ).action = 'ADD'
col.operator( "cutmask.list_action", icon='ZOOMOUT', text="" ).action = 'REMOVE'
col.separator()
col.operator( "cutmask.list_action", icon='TRIA_UP', text="" ).action = 'UP'
col.operator( "cutmask.list_action", icon='TRIA_DOWN', text="" ).action = 'DOWN'
layout.row().prop( scn, "cutMaskDefaultColor", text="Default color" )
layout.row().prop( scn, "cutMaskUseActive", text="Active object as target" )
if not scn.cutMaskUseActive:
layout.row().prop_search( scn, "cutMaskObject", scn, "objects", text="Target" )
obj = None
try:
obj = scn.objects[scn.cutMaskObject]
except:
pass
if obj:
layout.row().prop_search( scn, "cutMaskObjectUVMap", obj.data, "uv_layers", text="UV map" )
else:
layout.row().prop( scn, "cutMaskObjectUVMap", text="UV map" )
layout.row().prop_search( scn, "cutMaskImage", bpy.data, "images", text="Mask" )
layout.row().prop( scn, "cutMaskDistance", text="UV margin" )
layout.row().operator( "cutmask.apply", text="Apply" )
if len( scn.cutMaskError ) > 0:
pass
class CutMaskListProp( bpy.types.PropertyGroup ):
name = bpy.props.StringProperty()
color = bpy.props.FloatVectorProperty( subtype = 'COLOR', default = (1.0, 1.0, 1.0), min = 0.0, max = 1.0, description = "color picker" )
def register():
bpy.utils.register_module(__name__)
bpy.types.Scene.cutMaskUseActive = bpy.props.BoolProperty()
bpy.types.Scene.cutMaskObject = bpy.props.StringProperty()
bpy.types.Scene.cutMaskObjectUVMap = bpy.props.StringProperty()
bpy.types.Scene.cutMaskImage = bpy.props.StringProperty()
bpy.types.Scene.cutMasks = bpy.props.CollectionProperty( type = CutMaskListProp )
bpy.types.Scene.cutMaskIndex = bpy.props.IntProperty()
bpy.types.Scene.cutMaskDistance = bpy.props.IntProperty( default = 1, min = 0 )
bpy.types.Scene.cutMaskDefaultColor = bpy.props.FloatVectorProperty( subtype = 'COLOR', default = (1.0, 1.0, 1.0), min = 0.0, max = 1.0, description = "color picker" )
bpy.types.Scene.cutMaskError = bpy.props.StringProperty()
def unregister():
bpy.utils.unregister_module(__name__)
del bpy.types.Scene.cutMaskObject
del bpy.types.Scene.cutMaskObjectUVMap
del bpy.types.Scene.cutMaskImage
del bpy.types.Scene.cutMasks
del bpy.types.Scene.cutMaskIndex
del bpy.types.Scene.cutMaskDistance
del bpy.types.Scene.cutMaskDefaultColor
if __name__ == "__main__":
register()