How to use python to query database in parallel How to use python to query database in parallel database database

How to use python to query database in parallel


Here is a multi-threaded code that does what you're trying to accomplish:

from threading import Thread, Lockclass DatabaseWorker(Thread):    __lock = Lock()    def __init__(self, db, query, result_queue):        Thread.__init__(self)        self.db = db        self.query = query        self.result_queue = result_queue    def run(self):        result = None        logging.info("Connecting to database...")        try:            conn = connect(host=host, port=port, database=self.db)            curs = conn.cursor()            curs.execute(self.query)            result = curs            curs.close()            conn.close()        except Exception as e:            logging.error("Unable to access database %s" % str(e))        self.result_queue.append(result)delay = 1result_queue = []worker1 = DatabaseWorker("db1", "select something from sometable",        result_queue)worker2 = DatabaseWorker("db1", "select something from othertable",        result_queue)worker1.start()worker2.start()# Wait for the job to be donewhile len(result_queue) < 2:    sleep(delay)job_done = Trueworker1.join()worker2.join()