import tensorflow as tf
from keras.applications import Xception
from keras.utils import multi_gpu_model
import numpy as np

num_samples = 1000
height = 224
width = 224
num_classes = 1000

# 實例化基礎模型（或者「模版」模型）。
# 我們推薦在 CPU 設備范圍內做此操作，
# 這樣模型的權重就會存儲在 CPU 內存中。
# 否則它們會存儲在 GPU 上，而完全被共享。
with tf.device('/cpu:0'):
 model = Xception(weights=None,
   input_shape=(height, width, 3),
   classes=num_classes)

# 復制模型到 8 個 GPU 上。
# 這假設你的機器有 8 個可用 GPU。
parallel_model = multi_gpu_model(model, gpus=8)
parallel_model.compile(loss='categorical_crossentropy',
   optimizer='rmsprop')

# 生成虛擬數據
x = np.random.random((num_samples, height, width, 3))
y = np.random.random((num_samples, num_classes))

# 這個 `fit` 調用將分布在 8 個 GPU 上。
# 由于 batch size 是 256, 每個 GPU 將處理 32 個樣本。
parallel_model.fit(x, y, epochs=20, batch_size=256)

# 通過模版模型存儲模型（共享相同權重）：
model.save('my_model.h5')

class ParallelModelCheckpoint(ModelCheckpoint):
 def __init__(self,model,filepath, monitor='val_loss', verbose=0,
   save_best_only=False, save_weights_only=False,
   mode='auto', period=1):
 self.single_model = model
 super(ParallelModelCheckpoint,self).__init__(filepath, monitor, verbose,save_best_only, save_weights_only,mode, period)

 def set_model(self, model):
 super(ParallelModelCheckpoint,self).set_model(self.single_model)

checkpoint = ParallelModelCheckpoint(original_model)

# Model where a shared LSTM is used to encode two different sequences in parallel
input_a = keras.Input(shape=(140, 256))
input_b = keras.Input(shape=(140, 256))

shared_lstm = keras.layers.LSTM(64)

# Process the first sequence on one GPU
with tf.device_scope('/gpu:0'):
 encoded_a = shared_lstm(tweet_a)
# Process the next sequence on another GPU
with tf.device_scope('/gpu:1'):
 encoded_b = shared_lstm(tweet_b)

# Concatenate results on CPU
with tf.device_scope('/cpu:0'):
 merged_vector = keras.layers.concatenate([encoded_a, encoded_b],
      axis=-1)

server = tf.train.Server.create_local_server()
sess = tf.Session(server.target)

from keras import backend as K
K.set_session(sess)