I am trying to hunt down a tutorial or example of texture painting vie a script. I have built a text-to-3d colab projects which generates images of a character from different angles then samples those images to create a point-e cloud and mesh, when importing the mesh .ply its is all one mesh but not painted, the point cloud also generated does contain the colour information, I have another basic but quite slow modal script which then generates ico spheres for each point with the correct colour. However what i would really like to do is use this point cloud information to paint the mesh, it seemed simple enough in my head but i cant seem to find an example of actual texture painting thats of any use, does anyone know of some good examples of this, or able to help would be amazing, perhaps im even missing a trick as the point-e documentation says it can create the coloured mesh for you, eg here... https://github.com/openai/point-e?tab=readme-ov-file
ive included the bulk of the colab script which handles the text-to-image using gemini imagen 3 and the mesh generation, ive also included my simple blender script which runs on the point cloud mesh and generates the coloured balls correctly, any help or ideas on how to use the same vertex and colour data to paint the meshes would be helpfull, please see scripts below and screenshots to help visualize the method i was going to use...
partial colab script:
from vertexai.preview.vision_models import ImageGenerationModel
imagen_model = ImageGenerationModel.from_pretrained("imagen-3.0-fast-generate-001")
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('GPU')
try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
print("Memory growth enabled for GPU 0")
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
from PIL import Image
import torch
from tqdm.auto import tqdm
import numpy as np
import os
import subprocess
import json
from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config
from point_e.diffusion.sampler import PointCloudSampler
from point_e.util.pc_to_mesh import marching_cubes_mesh
from point_e.util.plotting import plot_point_cloud
from point_e.util.point_cloud import PointCloud
from point_e.models.download import load_checkpoint
from point_e.models.configs import MODEL_CONFIGS, model_from_config
from google.colab import userdata
image_prompt = input("please enter a character description?")
prompt_prefix = 'generate a character design sheet, this sheet shows a full body view of a character on a pure white background with no background shadows. Proffesional studio quality level of workmanship and super realistic character against a plain white background with no distractions, the character is always shown as a full body with arms out to the side in a typical A-pose. The character is : '
prompt_suffix = '. The character is standing with arms spread out to the side in the typical A pose. The character is of the highest possible detail against a blank white background, 4k textures, ultra-realistic materials, ultra-high resolution, fine-art-detailing close attention to muscle toning, character against a plain white background, no-blur.'
final_prompt = prompt_prefix + image_prompt + prompt_suffix
print(final_prompt)
view_angles = ["IMPORTANT: full-body front view", "IMPORTANT: full-body side view", "IMPORTANT: full-body back view"] # Define view angles
image_paths = [] # Store paths to generated images
for angle in view_angles:
prompt_with_angle = f"{final_prompt}, {angle}, full body" # Add angle to prompt
response = imagen_model.generate_images(
prompt=prompt_with_angle,
)
image = response.images[0]
image_path = f"./generated_image_{angle.replace(' ', '_')}.png" # Unique filename
image.save(image_path)
image_paths.append(image_path) # Add path to list
image.show()
print('please wait generating 3d model from images...')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Creating base model")
base_name = "base1B" # Use base1B for better results
base_model = model_from_config(MODEL_CONFIGS[base_name], device)
base_model.eval()
base_diffusion = diffusion_from_config(DIFFUSION_CONFIGS[base_name])
print('creating upsample model...')
upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device)
upsampler_model.eval()
upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample'])
print('downloading base checkpoint...')
base_model.load_state_dict(load_checkpoint(base_name, device))
print('downloading upsampler checkpoint...')
upsampler_model.load_state_dict(load_checkpoint('upsample', device))
sampler = PointCloudSampler(
device=device,
models=[base_model, upsampler_model],
diffusions=[base_diffusion, upsampler_diffusion],
num_points=[1024, 4096 - 1024],
aux_channels=['R', 'G', 'B'],
guidance_scale=[3.0, 0.0]
)
#image_tensor = torch.from_numpy(np.array(image)).float() / 255.0
#image_tensor = image_tensor.permute(2, 0, 1).unsqueeze(0) # Rearrange to [1, 3, H, W]
# Prepare image for CLIP model
#image_for_clip = (image_tensor.squeeze(0).permute(1, 2, 0) * 255).byte().cpu().numpy()
#image_for_clip = Image.fromarray(image_for_clip)
input_images = []
clip_images = []
for image_path in image_paths:
input_image = Image.open(image_path).convert("RGBA")
data = input_image.getdata()
newData = []
for item in data:
if item[0] == 255 and item[1] == 255 and item[2] == 255:
newData.append((255, 255, 255, 0))
else:
newData.append(item)
input_image.putdata(newData)
input_image.save(image_path)
input_images.append(input_image)
# Convert the image into a tensor-like object and add a batch dimension
image_tensor = torch.from_numpy(np.array(input_image)).float() / 255.0
image_tensor = image_tensor.permute(2, 0, 1).unsqueeze(0) # Rearrange to [1, 3, H, W]
# Prepare image for CLIP model
image_for_clip = (image_tensor.squeeze(0).permute(1, 2, 0) * 255).byte().cpu().numpy()
image_for_clip = Image.fromarray(image_for_clip)
clip_images.append(image_for_clip)
# Get the number of steps (iterations) for each model
base_num_steps = base_diffusion.num_timesteps
upsample_num_steps = upsampler_diffusion.num_timesteps
total_steps = 130
print(f"Total number of steps: {total_steps} (Base: {base_num_steps}, Upsampler: {upsample_num_steps})")
# Generate the point cloud from the images
print("Generating point cloud...")
samples = None
current_step = 0
with torch.no_grad():
# Loop through each image and generate a point cloud separately
for image_for_clip in tqdm(clip_images, total=len(clip_images), desc="Processing Images"):
for x in tqdm(
sampler.sample_batch_progressive(batch_size=1, model_kwargs=dict(images=[image_for_clip])), # Pass a single image at a time
total=total_steps,
desc="Sampling"
):
samples = x
current_step += 1
print(f"Completed {current_step}/{total_steps} steps")
# Inspect the structure of `samples`
print(f"samples: {samples}")
# Extract the point cloud from `samples`
try:
print('creating SDF model...')
name = 'sdf'
model = model_from_config(MODEL_CONFIGS[name], device)
model.eval()
print('loading SDF model...')
model.load_state_dict(load_checkpoint(name, device))
point_clouds = sampler.output_to_point_clouds(samples)
if isinstance(point_clouds, list) and len(point_clouds) > 0:
point_cloud = point_clouds[0] # Get the first point cloud
fig = plot_point_cloud(point_cloud, grid_size=2)
# Produce a mesh (with vertex colors)
mesh = marching_cubes_mesh(
pc=point_cloud,
model=model,
batch_size=4096,
grid_size=120, # increase to 128 for resolution used in evals
progress=True,
)
# Write the mesh to a PLY file to import into some other program.
with open('mesh.ply', 'wb') as f:
mesh.write_ply(f)
# Write the mesh to a GLB file
mesh.write_glb('mesh.glb')
else:
raise ValueError("No valid point clouds generated.")
except Exception as e:
print(f"Error extracting point cloud: {e}")
exit()
# Manual function to save the point cloud to a PLY file
def save_point_cloud_ply(filename, point_cloud):
try:
points = point_cloud.coords # No need for transpose, directly use the shape (num_points, 3)
colors = point_cloud.channels # Contains 'R', 'G', 'B' channels as separate arrays
# Start writing PLY file
with open(filename, 'w') as f:
# Write the header
f.write("ply\n")
f.write("format ascii 1.0\n")
f.write(f"element vertex {points.shape[0]}\n") # Number of points
f.write("property float x\n")
f.write("property float y\n")
f.write("property float z\n")
f.write("property uchar red\n")
f.write("property uchar green\n")
f.write("property uchar blue\n")
f.write("end_header\n")
# Write points and color
for i in range(points.shape[0]):
x, y, z = points[i, :] # Correctly index the points
r = int(colors['R'][i] * 255) # Convert from float to int (0-255)
g = int(colors['G'][i] * 255)
b = int(colors['B'][i] * 255)
f.write(f"{x} {y} {z} {r} {g} {b}\n")
except Exception as e:
print(f"Error saving point cloud: {e}")
# Save the point cloud to a PLY file
output_file = "output_point_cloud.ply"
try:
save_point_cloud_ply(output_file, point_cloud)
print(f"Point cloud saved to {output_file}")
except Exception as e:
print(f"Error saving point cloud: {e}")
print(f"dict contents = {point_cloud} failed to write to {output_file}")
blender script to visualize colored point cloud:
import bpy
import bmesh
from mathutils import Vector
from math import inf
class VERTEX_OT_VisualizeModalOperator(bpy.types.Operator):
bl_idname = "vertex.visualize_modal_operator"
bl_label = "Visualize Vertices (Modal)"
_timer = None
_bm = None
_vertex_idx = 0
_sorted_verts = []
def modal(self, context, event):
pblack = True
if event.type == 'ESC': # Cancel on pressing Escape
self.cancel(context)
return {'CANCELLED'}
if event.type == 'TIMER': # Process one vertex per timer event
if self._vertex_idx < len(self._sorted_verts):
original_index, vertex = self._sorted_verts[self._vertex_idx]
co = vertex.co
# Default to white if no color found
r, g, b = (1.0, 1.0, 1.0)
a = 1.0 # Alpha default
# Get color from the `Col` attribute using the original index
col_attr = self.obj.data.attributes.get('Col')
if col_attr:
color = col_attr.data[original_index].color
r, g, b, a = color[0], color[1], color[2], color[3] # RGBA values
else:
print(f"Vertex {self._vertex_idx}: Position = {co}, No color attribute available.")
print(f" Data: co={vertex.co}, normal={vertex.normal}, groups={vertex.groups}")
# Find the radius by measuring distance to the closest neighbor
radius = self.calculate_closest_distance(vertex)
if col_attr:
color = col_attr.data[original_index].color
r, g, b, a = color[0], color[1], color[2], color[3] # RGBA values
if r > 0.02 or g > 0.025 or b > 0.025 or pblack is True:
# Create an icosphere at the vertex position
bpy.ops.mesh.primitive_ico_sphere_add(
subdivisions=1,
radius=radius,
enter_editmode=False,
align='WORLD',
location=co
)
ico_sphere = bpy.context.active_object
# Create and assign material to the icosphere
matName = f'Vertex_Mat_{r}_{g}_{b}'
mat = self.get_or_create_material(matName, r, g, b, a)
ico_sphere.data.materials.append(mat)
# Move to the next vertex
self._vertex_idx += 1
else:
# End modal operation when all vertices are processed
self.report({'INFO'}, "Visualization complete.")
self.cancel(context)
return {'FINISHED'}
return {'RUNNING_MODAL'}
def execute(self, context):
# Initial setup for modal operation
self.obj = context.object
if self.obj is None or self.obj.type != 'MESH':
self.report({'ERROR'}, "Please select a mesh object.")
return {'CANCELLED'}
if bpy.context.mode != 'OBJECT':
bpy.ops.object.mode_set(mode='OBJECT')
mesh = self.obj.data
self._bm = bmesh.new() # Create a new BMesh object
self._bm.from_mesh(mesh) # Load the object's mesh data into BMesh
# Ensure lookup table is up-to-date to access vertices by index
self._bm.verts.ensure_lookup_table()
# Ensure the mesh has a `Col` attribute
if not mesh.attributes.get('Col'):
self.report({'ERROR'}, "No 'Col' attribute found in the mesh.")
return {'CANCELLED'}
# Sort vertices by their Y-coordinate and keep track of original indices
self._sorted_verts = sorted(enumerate(self._bm.verts), key=lambda v: v[1].co.z)
self._vertex_idx = 0 # Start at the first vertex
# Start modal timer
wm = context.window_manager
self._timer = wm.event_timer_add(0.1, window=context.window)
wm.modal_handler_add(self)
return {'RUNNING_MODAL'}
def cancel(self, context):
# Cleanup after modal operation
wm = context.window_manager
wm.event_timer_remove(self._timer)
if self._bm is not None:
self._bm.free() # Free the BMesh object to avoid memory leaks
self._vertex_idx = 0
def get_or_create_material(self, name, r, g, b, a):
"""Get or create a material with the given color."""
if name in bpy.data.materials:
return bpy.data.materials[name]
# Create a new material if it doesn't exist
mat = bpy.data.materials.new(name=name)
mat.use_nodes = True
bsdf = mat.node_tree.nodes.get('Principled BSDF')
if bsdf:
bsdf.inputs['Base Color'].default_value = (r, g, b, a) # RGBA
return mat
def calculate_closest_distance(self, current_vertex):
"""Calculate the distance from the current vertex to the closest neighboring vertex."""
current_position = current_vertex.co
min_distance = inf
for vertex in self._bm.verts:
if vertex != current_vertex: # Skip itself
dist = (current_position - vertex.co).length
if dist < min_distance:
min_distance = dist
return min_distance
# Define a new panel in the 3D View > Tool Shelf (N panel) to hold the UI
class VERTEX_PT_VisualizerPanel(bpy.types.Panel):
bl_label = "Vertex Visualizer"
bl_idname = "VERTEX_PT_visualizer_panel"
bl_space_type = 'VIEW_3D'
bl_region_type = 'UI'
bl_category = 'Vertex Viz'
def draw(self, context):
layout = self.layout
scene = context.scene
# Add a float input for vertex size (V size)
layout.prop(scene.vertex_visualizer_props, "v_size")
# Add the button to start the modal operator
layout.operator("vertex.visualize_modal_operator", text="Visualize (Modal)")
# Define custom properties for vertex size
class VertexVisualizerProperties(bpy.types.PropertyGroup):
v_size: bpy.props.FloatProperty(
name="V size",
description="Size of each icosphere representing a vertex",
default=0.005,
min=0.001,
max=9999.0
)
# Register and Unregister all the classes and properties
classes = [
VERTEX_OT_VisualizeModalOperator,
VERTEX_PT_VisualizerPanel,
VertexVisualizerProperties
]
def register():
for cls in classes:
bpy.utils.register_class(cls)
bpy.types.Scene.vertex_visualizer_props = bpy.props.PointerProperty(type=VertexVisualizerProperties)
def unregister():
for cls in classes:
bpy.utils.unregister_class(cls)
del bpy.types.Scene.vertex_visualizer_props
if __name__ == "__main__":
register()
mesh and coloured point cloud images:
