How do I Download an External File from URL and Display Progress in a Panel without blocking the UI?

Question

I've read the following answers already on SE, but I cannot yet seem to replicate the end result I need in Blender 2.71.

• https://stackoverflow.com/questions/13485720/blender-python-scripting-trying-to-prevent-ui-lock-up-while-doing-large-calcula/16744008#16744008
• Blender not responding while running my script
• Blender UI Multithreading Progressbar

I'm running a panel script that uses an operator button to download a file from an external URL. I've not yet found a way to do this without blocking the UI.

I've tested modal operators and used multithreading, no luck yet. Can anyone demonstrate that this is actually possible?

If it is possible, I would then use a modal operator timer to check on the progress and update a label in the ui panel.

Here's my attempt via a modal operator timer similar to the first link in the list.

import bpy
import sys
import urllib
from urllib import request
from urllib.request import urlretrieve


def reporthook(blocknum, blocksize, totalsize):
    readsofar = blocknum * blocksize
    if totalsize > 0:
        percent = readsofar * 1e2 / totalsize
        s = "\r%5.1f%% %*d / %d" % (percent, len(str(totalsize)), readsofar, totalsize)
        sys.stderr.write(s)
        if readsofar >= totalsize: # near the end
            sys.stderr.write("\n")
    else: # total size is unknown
        sys.stderr.write("read %d\n" % (readsofar,))



class ModalTimerOperator(bpy.types.Operator):
    """Operator which runs its self from a timer"""
    bl_idname = "wm.modal_timer_operator"
    bl_label = "Modal Timer Operator"

    _timer = None
    _updating = False
    _download_done = False

    def do_download(self):
        print("starting file download")

        #actual url and local path removed for privacy
        urlretrieve("REMOTE_URL", "LOCAL_DEST", reporthook) 

        self._download_done = True


    def modal(self, context, event):

        if event.type == 'TIMER' and not self._updating:
            self._updating = True
            self.do_download()
            self._updating = False

        if self._download_done:
            self.cancel(context)

        return {'PASS_THROUGH'}

    def execute(self, context):
        wm = context.window_manager
        self._timer = wm.event_timer_add(2, context.window)
        wm.modal_handler_add(self)
        self._updating = False
        return {'RUNNING_MODAL'}

    def cancel(self, context):
        wm = context.window_manager
        wm.event_timer_remove(self._timer)


def register():

    bpy.utils.register_class(ModalTimerOperator)

def unregister():
    bpy.utils.unregister_class(ModalTimerOperator)    
    bpy.utils.unregister_module(__name__)


if __name__ == "__main__":
    register()        



bpy.ops.wm.modal_timer_operator()

Answer 1

Okay, I think I've found the solution to this issue.

Instead of calling a single download file command with urlretrieve, I opened a remote urllib.request using the chunk size argument, as well as opening a local file for writing. I used a modal operator to read a new chunk from the URL and write the chunk to the file. Keeping the chunk size around 16kB seemed to work well.

In order to get the download size progress updated to a panel label, I needed to use a second modal operator running to get the label to update.

Curiously, in the Event.Timer statement of the second MO, I needed to update any random property in the scene context to get the Status Label to really update properly. Perhaps someone could explain that bit further.

This is what I used to trigger the update in the second MO:

bpy.context.window_manager.name = bpy.context.window_manager.name

Here's my code sample, feel free to comment:

import bpy
import sys
import urllib
from urllib import request

scn = bpy.context.scene
fileDwnProgress = ""
url = "REMOTEFILEURL"
theFile = "LOCALFILEPATH"

def strMsgLabel_Get(self):
    global fileDwnProgress
    return fileDwnProgress

bpy.types.Scene.strMsgLabel = bpy.props.StringProperty(
    name="Status", 
    description="Message Label", 
    get=strMsgLabel_Get, 
    set=None)


class OBJECT_PT_TESTPANEL(bpy.types.Panel):
    bl_label = "DOWNLOAD TEST PANEL"
    bl_space_type = "NODE_EDITOR"
    bl_region_type = "UI"
    bl_context = "object"

    def draw_header(self, context):
        layout = self.layout
        layout.label(text="", icon="INFO")

    def draw(self, context):
        layout = self.layout
        row = layout.row()
        row.label(text=scn.strMsgLabel)


class mdFileDownload(bpy.types.Operator):

    bl_idname = "wm.md_file_download"
    bl_label = "File Download"

    _timer = None
    _transferring = False
    _download = False
    _chunkSize = 16 * 1024  #(16kB seems good chunk size)
    _fileSize = 0
    _fp = None
    _req = None
    _chunk = None

    def modal(self, context, event):

        global fileDwnProgress
        global url 
        global theFile

        if event.type == 'TIMER' and not self._transferring and not self._download:

            self._fp = open(theFile, 'wb')
            self._req = urllib.request.urlopen(url)
            self._transferring = True

        if event.type == 'TIMER' and self._transferring:

            self._chunk = self._req.read(self._chunkSize)

            if sys.getsizeof(self._chunk) < self._chunkSize: 
                self._fp.write(self._chunk)
                self._fileSize += self._chunkSize
                self._transferring = False
                self._download = True  
                self._fp.close()
            else: 
                self._fp.write(self._chunk)
                self._fileSize += self._chunkSize

            progress = str(round((self._fileSize/1024/1024),1)) + " MB"
            fileDwnProgress = progress


        if self._download:
            fileDwnProgress = "Download complete!"
            self.cancel(context)

        return {'PASS_THROUGH'}


    def execute(self, context):
        # print(var)
        wm = context.window_manager
        self._timer = wm.event_timer_add(0.01, context.window)
        wm.modal_handler_add(self)
        self._transferring = False
        return {'RUNNING_MODAL'}

    def cancel(self, context):
        wm = context.window_manager
        wm.event_timer_remove(self._timer)



class mdDownloadMonitor(bpy.types.Operator):

    bl_idname = "wm.md_download_monitor"
    bl_label = "Download progress monitor"

    _timer = None

    def modal(self, context, event):

        if event.type == 'TIMER':
            #this is required to trigger the update
            bpy.context.window_manager.name = bpy.context.window_manager.name

        return {'PASS_THROUGH'}

    def execute(self, context):
        wm = context.window_manager
        self._timer = wm.event_timer_add(0.01, context.window)
        wm.modal_handler_add(self)
        bpy.ops.wm.md_file_download()
        return {'RUNNING_MODAL'}

    def cancel(self, context):
        wm = context.window_manager
        wm.event_timer_remove(self._timer)



def register():

    bpy.utils.register_class(OBJECT_PT_TESTPANEL)
    bpy.utils.register_class(mdFileDownload)
    bpy.utils.register_class(mdDownloadMonitor)

def unregister():
    bpy.utils.unregister_class(OBJECT_PT_TESTPANEL)
    bpy.utils.unregister_class(mdFileDownload) 
    bpy.utils.unregister_class(mdDownloadMonitor)
    bpy.utils.unregister_module(__name__)

if __name__ == "__main__":
    register()        

bpy.ops.wm.md_download_monitor()