Python中運(yùn)行并行任務(wù)技巧
示例
標(biāo)準(zhǔn)線程多進(jìn)程,生產(chǎn)者/消費(fèi)者示例:
Worker越多,問題越大
# -*- coding: utf8 -*-
import os
import time
import Queue
import threading
from PIL import Image
def create_thumbnail(filename, size=(128, 128)):
try:
fp, fmt = filename.rsplit('.', 1)
im = Image.open(filename)
im.thumbnail(size, Image.ANTIALIAS)
im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
return '%s thumbnail success!' % filename
except Exception:
return '%s thumbnail failed!' % filename
def get_image_paths(folder):
return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]
class Consumer(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self._queue = queue
def run(self):
while True:
content = self._queue.get()
if isinstance(content, str) and content == 'quit':
break
respone = create_thumbnail(content)
print 'Bye bye!'
def Producer():
filenames = get_image_paths('images')
queue = Queue.Queue()
worker_threads = build_worker_pool(queue, 4)
start_time = time.time()
for filename in filenames:
queue.put(filename)
for worker in worker_threads:
queue.put('quit')
for worker in worker_threads:
worker.join()
print time.time() - start_time
def build_worker_pool(queue, size):
workers = []
for _ in range(size):
worker = Consumer(queue)
worker.start()
workers.append(worker)
return workers
if __name__ == '__main__':
Producer()
map
Map能夠處理集合按順序遍歷,最終將調(diào)用產(chǎn)生的結(jié)果保存在一個(gè)簡單的集合當(dāng)中。
# -*- coding: utf8 -*-
import os
import time
from multiprocessing import Pool
from PIL import Image
def create_thumbnail(filename, size=(128, 128)):
try:
fp, fmt = filename.rsplit('.', 1)
im = Image.open(filename)
im.thumbnail(size, Image.ANTIALIAS)
im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
return '%s thumbnail success!' % filename
except Exception:
return '%s thumbnail failed!' % filename
def get_image_paths(folder):
return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]
def main():
filenames = get_image_paths('images')
start_time = time.time()
pool = Pool(4)
pool.map(create_thumbnail, filenames)
pool.close()
pool.join()
print time.time() - start_time
if __name__ == '__main__':
main()
相關(guān)文章
關(guān)于python中readlines函數(shù)的參數(shù)hint的相關(guān)知識總結(jié)
今天給大家?guī)淼氖顷P(guān)于Python函數(shù)的相關(guān)知識,文章圍繞著python中readlines函數(shù)的參數(shù)hint展開,文中有非常詳細(xì)的介紹及代碼示例,需要的朋友可以參考下2021-06-06Python利用Flask-Mail實(shí)現(xiàn)發(fā)送郵件詳解
Flask?的擴(kuò)展包?Flask?-?Mail?通過包裝了?Python?內(nèi)置的smtplib包,可以用在?Flask?程序中發(fā)送郵件。本文將利用這特性實(shí)現(xiàn)郵件發(fā)送功能,感興趣的可以了解一下2022-08-08tensorflow學(xué)習(xí)筆記之mnist的卷積神經(jīng)網(wǎng)絡(luò)實(shí)例
這篇文章主要為大家詳細(xì)介紹了tensorflow學(xué)習(xí)筆記之mnist的卷積神經(jīng)網(wǎng)絡(luò)實(shí)例,具有一定的參考價(jià)值,感興趣的小伙伴們可以參考一下2018-04-04利用python實(shí)現(xiàn)簡單的郵件發(fā)送客戶端示例
下面小編就為大家分享一篇利用python實(shí)現(xiàn)簡單的郵件發(fā)送客戶端示例,具有很好的參考價(jià)值,希望對大家有所幫助。一起跟隨小編過來看看吧2017-12-12