That is done in XSI's ICE, which has an inbuilt function for retrieving colour info via raycasting
I came up with the following code, based on these posts:
import bpy from bpy import context #for BVHTree dependency graph from os import system #for clearing console import numpy as np # for interpreting image as np.array from PIL import Image # open image from mathutils import Vector # for constructing the BVHTree from mathutils.geometry import barycentric_transform #for finding the coresponding point in the uv space (denoted b_point) for a point of the mesh (denoted hit_location) from mathutils.bvhtree import BVHTree as bvh # for ray casting '''Mesh has to be triangulated. Possible alternative algorithmic triangulation using scipy.spatial.delaunay. To be investigated ''' # debug purposes, clears some scene elements and the console cls = lambda: system('cls') cls() objs = bpy.data.objects if len(objs) > 1: objs.remove(objs["Cube"], do_unlink=True) objs.remove(objs["Cube.001"], do_unlink=True) #get object to be raycasted on, needs UVs obj = bpy.context.scene.objects["Plane"] #build BVHTree + dependency graph depsgraph = context.evaluated_depsgraph_get() shoot_ray = bvh.FromObject(obj, depsgraph) #defines BVHTree args ray_origin = Vector((-.12,.24,0.1)) direction = Vector((0.0,0.0,-1)) distance = 6 #returns the BVHTree raycast data hit_location, hit_normal, face_index, distance = shoot_ray.ray_cast(ray_origin, direction, distance) #hit_normal,distance unused print(face_index) # debug purposes, creates a cube at ray origin bpy.ops.mesh.primitive_cube_add(enter_editmode=False, align='WORLD', location=ray_origin, scale=(.01, .01, .01)) material = bpy.data.materials["ray_loc"] cube = bpy.data.objects['Cube'] cube.active_material = material #from the face's index calculated by the BVHTree finds the coresponding vertices as list verts_indices = obj.data.polygons[face_index].vertices #decompose vertices' list in individial elements vert1, vert2, vert3 = [obj.data.vertices[verts_indices[i]].co for i in range(3)] #from the face's index calculated by the BVHTree finds the coresponding UVs as a list uvMap_indices = obj.data.polygons[face_index].loop_indices #for the lookup, gets the UV map in use uvMap = obj.data.uv_layers['UVMap'] #decompose the UVs list in individual components uv_1, uv_2, uv_3 = [uvMap.data[uvMap_indices[i]].uv for i in range(3)] #conversion of the UV locations to a 3D vector, as the barycentric calculation uses an more generic implementation (3D), z will be 0 uv1 = uv_1.to_3d() uv2 = uv_2.to_3d() uv3 = uv_3.to_3d() #barycentric calculation of the coresponding point in the uv space b_point = barycentric_transform( hit_location, vert1, vert2, vert3, uv1, uv2, uv3 ) #reduces the 3d vector back to a 2d vector b_point.resize_2d() #loads image to be interpreted as a numpy array. The following alternative " image = bpy.data.images['v.jpg'] " is a possible solution, but it has to be first converted to a list, then to an np.array. image = Image.open('C:/Users/x/Desktop/v1.jpg') pixels = np.array(image) # gets the image dimensions width, height = image.size #gets the x,y coordinates of the pixel and finds the (ungefähr) pixel in the array. rounding errors are expected to occur uv_x = round(b_point*width) uv_y = round(b_point*height) rgb = pixels[uv_x][uv_y] #adds alpha and linearizes the rgba value rgba = np.append (rgb,1)/255 # debug purposes, adds a cube at the intersection location and changes the color of the cube to the raycasted color bpy.ops.mesh.primitive_cube_add(enter_editmode=False, align='WORLD', location=hit_location, scale=(.01, .01, .01)) material1 = bpy.data.materials["hit_loc"] material1.use_nodes = True principled_bsdf = material.node_tree.nodes['Principled BSDF'] principled_bsdf.inputs.default_value = tuple(rgba) cube1 = bpy.data.objects['Cube.001'] cube1.active_material = material1 #wrong result...why?
I commented almost every line of the code, so it should be easy to follow ( I will neatly organize it in functions after it will be functional). Now, the code seems logical to me and it works fine, only it returns the wrong colour. Something is obviously off, but I can't understand where my mistake is. Any suggestion greatly appreciated! Here is the .blend file and the image I used for debugging https://we.tl/t-vwwJrsW3IC