0
$\begingroup$

I try to bake a sound to fcurve in headless mode. I wrote some code which is quite close to.

    import bpy
    # Import the operator
    from bpy.ops import sound

    def switch_blender_area_OLD(area_type):
        areas  = [area for area in bpy.context.window.screen.areas if area.type == area_type]
        bpy.context.area.ui_type = area_type
        print("area_type", area_type)
        
    def switch_blender_area(area_type):
        # Iterate through the areas of the current screen and set the active area based on its type
        for area in bpy.context.screen.areas:
            if area.type == area_type:
                bpy.context.area = area
                    
    def hex_to_rgb( hex_value ):
        b = (hex_value & 0xFF) / 255.0
        g = ((hex_value >> 8) & 0xFF) / 255.0
        r = ((hex_value >> 16) & 0xFF) / 255.0
        return r, g, b, 1

    def setup_audio_visualisation_color(material_name, color_ramp_name, color_index, hex_color):
        bpy.data.materials[material_name].node_tree.nodes[color_ramp_name].color_ramp.elements[color_index].color = hex_to_rgb(hex_color)
        
    def reset_audio():
        # Remove all strips from the sequence editor
        for strip in bpy.context.scene.sequence_editor.sequences:
            bpy.context.scene.sequence_editor.sequences.remove(strip)

        # Add a new sound strip using the name of the sound
        bpy.context.scene.sequence_editor.sequences.new_sound(
            name = sound_name,
            filepath = audio_wav_filename,
            relative_path = False,
            frame_start = 1,
            channel = 1
        )

        # Set the end frame of the scene to the final frame of the sound strip
        for strip in bpy.context.scene.sequence_editor.sequences:
            bpy.context.scene.frame_end = strip.frame_final_end
            print(strip.frame_final_end, "frames")
        
    def setup_audio_visualisation_scene(audio_visualization_host_object_name, audio_visualization_modifier_name, audio_visualization_modifier_input_parametername, audio_wav_filename):
        # Get the host object and select it
        host_object = bpy.data.objects[audio_visualization_host_object_name]
        host_object.select_set(True)

        # Set the current frame to 1
        bpy.context.scene.frame_current = 1

        # Create a new action
        action = bpy.data.actions.new(name="AV")

        # Clear the animation data for the host object
        host_object.animation_data_clear()

        # Create an animation data object for the host object and link it to the action
        host_object.animation_data_create()
        host_object.animation_data.action = action


        # Create a new f-curve for the host object and set the initial value
        fcurve = host_object.animation_data.action.fcurves.new(
            data_path = 'modifiers["' + audio_visualization_modifier_name + '"]["' + audio_visualization_modifier_input_parametername + '"]',
            index = 0
        )
        fcurve.keyframe_points.insert(1, 1.0)

        # Select and activate the f-curve
        fcurve.select = True
        # fcurve.active = True

        # Bake the sound to the f-curve
        ##### bpy.ops.graph.sound_bake(filepath=audio_wav_filename)
        
        
        reset_audio()

    setup_audio_visualisation_scene(
        audio_visualization_host_object_name = "Cube", 
        audio_visualization_modifier_name = "AV", 
        audio_visualization_modifier_input_parametername = "Input_4", 
        audio_wav_filename = "C:\\Users\\steph\\Desktop\\test_tone.wav"
        #audio_wav_filename = "C:\\Users\\steph\\Desktop\\demo_mastered.wav"
    ) 

bpy.ops.graph.sound_bake(filepath=audio_wav_filename) is not possible in headless mode since I need to switch the context aka area.

So I thought about to implement it by myself like this prototype.

class AmplitudeAnalyzer:
    def __init__(self, audio_wav_filename, lowest_frequency=0, highest_frequency=100000, attack_time=0.005, release_time=0.2, threshold=0.1):
        self.audio_wav_filename = audio_wav_filename
        self.lowest_frequency = lowest_frequency
        self.highest_frequency = highest_frequency
        self.attack_time = attack_time
        self.release_time = release_time
        self.threshold = threshold
        
        self.audio_file = None
        self.audio_data = None
        self.num_channels = None
        self.frame_rate = None
        self.frame_length = None
        self.num_frames = None
        self.amplitudes = None
        
        # Open the audio file
        with wave.open(self.audio_wav_filename, 'rb') as self.audio_file:
            # Read the audio data from the file
            audio_data = self.audio_file.readframes(self.audio_file.getnframes())

            # Get the number of channels in the audio data
            self.num_channels = self.audio_file.getnchannels()
            
        # Convert the audio data to a NumPy array
        self.audio_data = np.frombuffer(audio_data, dtype=np.int16)

        # Calculate the frame rate of the audio data
        self.frame_rate = self.audio_file.getframerate()

        # Calculate the length of each frame in seconds
        self.frame_length = 1 / self.frame_rate

        # Calculate the number of frames in the audio data
        self.num_frames = len(self.audio_data) // self.num_channels

        # Initialize an array to hold the amplitudes for each frame
        self.amplitudes = np.zeros(self.num_frames)
        
    def get_num_frames_to_display(self, scene_frame_rate):
        # Calculate the length of the audio data in seconds
        audio_length_seconds = self.num_frames / self.frame_rate

        # Calculate the number of frames needed to display the complete waveform
        num_frames_to_display = int(audio_length_seconds * scene_frame_rate)
        
        return num_frames_to_display
    
    def get_amplitude_at_video_frame(self, video_frame_position):
        # Get the number of frames to display in the scene
        num_frames_to_display = self.get_num_frames_to_display(bpy.context.scene.render.fps)

        # Calculate the length of each frame in seconds
        frame_length = 1 / self.frame_rate

        # Calculate the index of the frame that corresponds to the given video frame position
        frame_index = int(len(self.amplitudes) / num_frames_to_display * video_frame_position)

        if (video_frame_position == 0 or video_frame_position == num_frames_to_display):
            return 0
        else:   
            # Check if the frame index is within the bounds of the amplitudes array
            if frame_index >= 0 and frame_index < len(self.amplitudes):
                # If the index is within bounds, return the corresponding amplitude value
                return self.amplitudes[frame_index]
            else:
                # If the index is out of bounds, return zero
                return 0



    def get_frame_data(self, i):
        # Calculate the start and end times for this frame
        frame_start_time = i * self.frame_length
        frame_end_time = frame_start_time + self.frame_length

        # Calculate the attack and release time for this frame in samples
        attack_samples = int(self.attack_time * self.frame_rate)
        release_samples = int(self.release_time * self.frame_rate)

        # Calculate the start and end indices for this frame in the audio data array
        frame_start_index = int(frame_start_time * self.frame_rate)
        frame_end_index = int(frame_end_time * self.frame_rate)
        
        return frame_start_time, frame_end_time, attack_samples, release_samples, frame_start_index, frame_end_index

    def build_amplitude_array(self):
        # Initialize an array to hold the amplitudes for each frame
        self.amplitudes = np.zeros(self.num_frames)

        # Loop through the audio data and populate the amplitudes array
        for i in range(self.num_frames):
            # Get the start and end times, attack and release samples, and start and end indices for this frame
            t1, t2, a, r, idx1, idx2 = self.get_frame_data(i)

            # Make a copy of the audio data array and convert it to a floating point array
            frame_amplitudes = self.audio_data[idx1:idx2].astype(np.float64)

            # Check if the length of the frame amplitudes array is zero
            if len(frame_amplitudes) == 0:
                # If the length is zero, set the corresponding entry in the amplitudes array to zero
                self.amplitudes[i] = 0
            else:
                # Calculate the envelope for this frame
                if i == 0:
                    # Calculate the attack coefficients for the first frame
                    attack_coeff = np.linspace(0, 1, a)
                    attack = attack_coeff[:a]
                    env = attack
                elif i == self.num_frames - 1:
                    # Calculate the release coefficients for the last frame
                    release_coeff = np.linspace(1, 0, r)
                    release = release_coeff[r:]
                    env = release
                else:
                    # Calculate the attack and release coefficients for all other frames
                    attack_coeff = np.linspace(0, 1, a)
                    attack = attack_coeff[:a]
                    release_coeff = np.linspace(1, 0, r)
                    release = release_coeff[r:]

                    # Calculate the length of the middle section of the envelope
                    middle_length = idx2 - idx1 - a - r

                    # Check if the length of the middle section is non-negative
                    if middle_length < 0:
                        # If the length is negative, set it to zero
                        middle_length = 0

                    # Concatenate the attack, middle, and release sections of the envelope
                    env = np.concatenate((attack, np.ones(middle_length), release))

                # Resize the frame amplitudes array to match the length of the envelope array
                frame_amplitudes = np.resize(frame_amplitudes, len(env))

                # Multiply the audio data array and the envelope array element-wise
                frame_amplitudes = frame_amplitudes * env

                # Calculate the RMS value of the envelope-weighted audio data
                rms = np.sqrt(np.mean(frame_amplitudes**2))

                # Store the RMS value in the amplitudes array
                self.amplitudes[i] = rms



        amplitude_analyzer = AmplitudeAnalyzer(
            audio_wav_filename=audio_wav_filename 
        )
        amplitude_analyzer.build_amplitude_array()

        # Set the end frame of the current scene to the length of the audio file
        num_frames_to_display = amplitude_analyzer.get_num_frames_to_display(bpy.context.scene.render.fps)
        bpy.context.scene.frame_end = num_frames_to_display 

        # Iterate over the audio samples
        for frame in range(num_frames_to_display + 1):
            if frame % 4 == 0:
                # Calculate the current frame number based on the audio's frame rate
                amp = amplitude_analyzer.get_amplitude_at_video_frame(frame)
                # Insert a keyframe at the current frame number using the sample value
                fcurve.keyframe_points.insert(frame, value=amp, options={'FAST'})

But without success so far. Pretty difficult to match data from wav file to current scene framerate and I also doubt it is trivial to implement features Blenders GUI provide.

enter image description here

My prototype python code creates these keyframes

enter image description here

but should look like this

enter image description here

Does anyone know a easy simple method to self bake audio data to curve which respects at least frequency filter?

$\endgroup$
4
  • $\begingroup$ i gotta ask, are the keyframes, the whole sound file? like have you tried expanding the frame limit then trying the code again, bc its weird it doesnt end like it starts $\endgroup$
    – Anity Ex
    Dec 8, 2022 at 18:09
  • $\begingroup$ Today i resolved the task with A LOT of math. I cannot delete a bounty question. No answer needed anymore. $\endgroup$ Dec 10, 2022 at 19:33
  • 3
    $\begingroup$ Why not answer your own question then? $\endgroup$ Dec 10, 2022 at 22:00
  • 1
    $\begingroup$ answer your own question to share the knowledge with others $\endgroup$ Dec 14, 2022 at 17:34

0

You must log in to answer this question.

Browse other questions tagged .