Python script use while loop to keep updating job scripts and multiprocess the tasks in queue

Where did you define sql_file in multiple_thread() in

multiprocessing.Process(target=query_sql, args=(sql_file)).start()

You have not defined sql_file in the method and moreover you have used that variable in a for loop. The variable's scope is only confined to the for loop.

Try replacing this:

result = func(*args)

by this:

result = func(args)

I have figured this out. Thank your for the response inspired the thought.Now the script can run a while loop to monitor the folder for new updated/added SQL script, and then distribute the data pulling to multiple threads. The solution comes from the queue.get(), and queue.put(). I assume the queue object takes care of the communication by itself.

This is the final code --

from glob import globimport os, timeimport sysimport pypyodbcfrom multiprocessing import Process, Queue, Event, Pool, current_process, freeze_supportdef query_sql(sql_file): #test func    #jsl file processing and SQL querying, data table will be saved to csv.    fo_name = os.path.splitext(sql_file)[0] + '.csv'    fo = open(fo_name, 'w')    print sql_file    fo.write("sql_file {0} is done\n".format(sql_file))    return "Query is done for \n".format(sql_file)def check_files(path):    """    arguments -- root path to monitor    returns   -- dictionary of {file: timestamp, ...}    """    sql_query_dirs = glob(path + "/*/IDABox/")    files_dict = {}    try:        for sql_query_dir in sql_query_dirs:            for root, dirs, filenames in os.walk(sql_query_dir):                [files_dict.update({(root + filename): os.path.getmtime(root + filename)}) for                          filename in filenames if filename.endswith('.jsl')]    except:        pass    return files_dictdef worker_main(queue):    print os.getpid(),"working"    while True:        item = queue.get(True)        query_sql(item)def main():    the_queue = Queue()    the_pool = Pool(4, worker_main,(the_queue,))    path = "Y:/"    before = check_files(path) # get the current dictionary of sql_files    while True:         #while loop to check the changes of the files        time.sleep(5)        sql_queue  = []         after = check_files(path)        added = [f for f in after if not f in before]        deleted = [f for f in before if not f in after]        overlapped = list(set(list(after)) & set(list(before)))        updated = [f for f in overlapped if before[f] < after[f]]          before = after          sql_queue = added + updated           if sql_queue:            for jsl_file in sql_queue:                try:                    the_queue.put(jsl_file)                except:                    print "{0} failed with error {1}. \n".format(jsl_file, str(sys.exc_info()[0]))                    pass        else:            passif __name__ == "__main__":    main()