keras 两种训练模型方式详解fit和fit_generator(节省内存)

落日映苍穹つ 2022-12-26 00:58 91阅读 0赞

更多编程教程请到：[菜鸟教程][Link 1] https://www.piaodoo.com/

友情链接：

[高州阳光论坛][Link 2]https://www.hnthzk.com/

[人人影视][Link 3]http://www.op-kg.com/

**第一种，fit**

import keras
    from keras.models import Sequential
    from keras.layers import Dense
    import numpy as np
    from sklearn.preprocessing import LabelEncoder
    from sklearn.preprocessing import OneHotEncoder
    from sklearn.model_selection import train_test_split

\#读取数据  
x\_train = np.load(“D:\\machineTest\\testmulPE\_win7\\data\_sprase.npy”)\[()\]  
y\_train = np.load(“D:\\machineTest\\testmulPE\_win7\\lable\_sprase.npy”)

# 获取分类类别总数 #

classes = len(np.unique(y\_train))

\#对label进行one-hot编码，必须的  
label\_encoder = LabelEncoder()  
integer\_encoded = label\_encoder.fit\_transform(y\_train)  
onehot\_encoder = OneHotEncoder(sparse=False)  
integer\_encoded = integer\_encoded.reshape(len(integer\_encoded), 1)  
y\_train = onehot\_encoder.fit\_transform(integer\_encoded)

\#shuffle  
X\_train, X\_test, y\_train, y\_test = train\_test\_split(x\_train, y\_train, test\_size=0.3, random\_state=0)

model = Sequential()  
model.add(Dense(units=1000, activation=‘relu’, input\_dim=784))  
model.add(Dense(units=classes, activation=‘softmax’))  
model.compile(loss=‘categorical\_crossentropy’,  
optimizer=‘sgd’,  
metrics=\[‘accuracy’\])  
model.fit(X\_train, y\_train, epochs=50, batch\_size=128)  
score = model.evaluate(X\_test, y\_test, batch\_size=128)

# \#fit参数详情 #

# keras.models.fit( #

# self, #

# x=None, \#训练数据 #

# y=None, \#训练数据label标签 #

# batch\_size=None, \#每经过多少个sample更新一次权重，defult 32 #

# epochs=1, \#训练的轮数epochs #

# verbose=1, \#0为不在标准输出流输出日志信息，1为输出进度条记录，2为每个epoch输出一行记录 #

# callbacks=None,\#list，list中的元素为keras.callbacks.Callback对象，在训练过程中会调用list中的回调函数 #

# validation\_split=0., \#浮点数0-1，将训练集中的一部分比例作为验证集，然后下面的验证集validation\_data将不会起到作用 #

# validation\_data=None, \#验证集 #

# shuffle=True, \#布尔值和字符串，如果为布尔值，表示是否在每一次epoch训练前随机打乱输入样本的顺序，如果为"batch"，为处理HDF5数据 #

# class\_weight=None, \#dict,分类问题的时候，有的类别可能需要额外关注，分错的时候给的惩罚会比较大，所以权重会调高，体现在损失函数上面 #

# sample\_weight=None, \#array,和输入样本对等长度,对输入的每个特征+个权值，如果是时序的数据，则采用(samples，sequence\_length)的矩阵 #

# initial\_epoch=0, \#如果之前做了训练，则可以从指定的epoch开始训练 #

# steps\_per\_epoch=None, \#将一个epoch分为多少个steps，也就是划分一个batch\_size多大，比如steps\_per\_epoch=10，则就是将训练集分为10份，不能和batch\_size共同使用 #

# validation\_steps=None, \#当steps\_per\_epoch被启用的时候才有用，验证集的batch\_size #

# \*\*kwargs \#用于和后端交互 #

# ) #

#  #

# 返回的是一个History对象，可以通过History.history来查看训练过程，loss值等等 #

**第二种，fit\_generator(节省内存)**

# 第二种,可以节省内存
    '''
    Created on 2018-4-11
    fit_generate.txt，后面两列为lable,已经one-hot编码
    1 2 0 1
    2 3 1 0
    1 3 0 1
    1 4 0 1
    2 4 1 0
    2 5 1 0

‘’’  
import keras  
from keras.models import Sequential  
from keras.layers import Dense  
import numpy as np  
from sklearn.model\_selection import train\_test\_split

count =1  
def generate\_arrays\_from\_file(path):  
global count  
while 1:  
datas = np.loadtxt(path,delimiter=’ ',dtype=“int”)  
x = datas\[:,:2\]  
y = datas\[:,2:\]  
print(“count:”+str(count))  
count = count+1  
yield (x,y)  
x\_valid = np.array(\[\[1,2\],\[2,3\]\])  
y\_valid = np.array(\[\[0,1\],\[1,0\]\])  
model = Sequential()  
model.add(Dense(units=1000, activation=‘relu’, input\_dim=2))  
model.add(Dense(units=2, activation=‘softmax’))  
model.compile(loss=‘categorical\_crossentropy’,  
optimizer=‘sgd’,  
metrics=\[‘accuracy’\])

model.fit\_generator(generate\_arrays\_from\_file(“D:\\fit\_generate.txt”),steps\_per\_epoch=10, epochs=2,max\_queue\_size=1,validation\_data=(x\_valid, y\_valid),workers=1)

# steps\_per\_epoch 每执行一次steps,就去执行一次生产函数generate\_arrays\_from\_file #

# max\_queue\_size 从生产函数中出来的数据时可以缓存在queue队列中 #

# 输出如下: #

# Epoch 1/2 #

# count:1 #

# count:2 #

#  #

# 1/10 \[==>…\] - ETA: 2s - loss: 0.7145 - acc: 0.3333count:3 #

# count:4 #

# count:5 #

# count:6 #

# count:7 #

#  #

# 7/10 \[====================>…\] - ETA: 0s - loss: 0.7001 - acc: 0.4286count:8 #

# count:9 #

# count:10 #

# count:11 #

#  #

# 10/10 \[==============================\] - 0s 36ms/step - loss: 0.6960 - acc: 0.4500 - val\_loss: 0.6794 - val\_acc: 0.5000 #

# Epoch 2/2 #

#  #

# 1/10 \[==>…\] - ETA: 0s - loss: 0.6829 - acc: 0.5000count:12 #

# count:13 #

# count:14 #

# count:15 #

#  #

# 5/10 \[==============>…\] - ETA: 0s - loss: 0.6800 - acc: 0.5000count:16 #

# count:17 #

# count:18 #

# count:19 #

# count:20 #

#  #

# 10/10 \[==============================\] - 0s 11ms/step - loss: 0.6766 - acc: 0.5000 - val\_loss: 0.6662 - val\_acc: 0.5000 #

**补充知识：**

自动生成数据还可以继承keras.utils.Sequence，然后写自己的生成数据类:

keras数据自动生成器，继承keras.utils.Sequence，结合fit\_generator实现节约内存训练

#coding=utf-8
    '''
    Created on 2018-7-10
    '''
    import keras
    import math
    import os
    import cv2
    import numpy as np
    from keras.models import Sequential
    from keras.layers import Dense

class DataGenerator(keras.utils.Sequence):

def **init**(self, datas, batch\_size=1, shuffle=True):  
self.batch\_size = batch\_size  
self.datas = datas  
self.indexes = np.arange(len(self.datas))  
self.shuffle = shuffle

def **len**(self):  
\#计算每一个epoch的迭代次数  
return math.ceil(len(self.datas) / float(self.batch\_size))

def **getitem**(self, index):  
\#生成每个batch数据，这里就根据自己对数据的读取方式进行发挥了

# 生成batch\_size个索引 #

batch\_indexs = self.indexes\[index\*self.batch\_size:(index+1)\*self.batch\_size\]

# 根据索引获取datas集合中的数据 #

batch\_datas = \[self.datas\[k\] for k in batch\_indexs\]

# 生成数据 #

X, y = self.data\_generation(batch\_datas)

return X, y

def on\_epoch\_end(self):  
\#在每一次epoch结束是否需要进行一次随机，重新随机一下index  
if self.shuffle == True:  
np.random.shuffle(self.indexes)

def data\_generation(self, batch\_datas):  
images = \[\]  
labels = \[\]

# 生成数据 #

for i, data in enumerate(batch\_datas):  
\#x\_train数据  
image = cv2.imread(data)  
image = list(image)  
images.append(image)  
\#y\_train数据  
right = data.rfind("\\",0)  
left = data.rfind("\\",0,right)+1  
class\_name = data\[left:right\]  
if class\_name==“dog”:  
labels.append(\[0,1\])  
else:  
labels.append(\[1,0\])  
\#如果为多输出模型，Y的格式要变一下，外层list格式包裹numpy格式是list\[numpy\_out1,numpy\_out2,numpy\_out3\]  
return np.array(images), np.array(labels)

# 读取样本名称，然后根据样本名称去读取数据 #

class\_num = 0  
train\_datas = \[\]  
for file in os.listdir(“D:/xxx”):  
file\_path = os.path.join(“D:/xxx”, file)  
if os.path.isdir(file\_path):  
class\_num = class\_num + 1  
for sub\_file in os.listdir(file\_path):  
train\_datas.append(os.path.join(file\_path, sub\_file))

# 数据生成器 #

training\_generator = DataGenerator(train\_datas)

\#构建网络  
model = Sequential()  
model.add(Dense(units=64, activation=‘relu’, input\_dim=784))  
model.add(Dense(units=2, activation=‘softmax’))  
model.compile(loss=‘categorical\_crossentropy’,  
optimizer=‘sgd’,  
metrics=\[‘accuracy’\])  
model.compile(optimizer=‘sgd’, loss=‘categorical\_crossentropy’, metrics=\[‘accuracy’\])

model.fit\_generator(training\_generator, epochs=50,max\_queue\_size=10,workers=1)

以上这篇keras 两种训练模型方式详解fit和fit\_generator(节省内存)就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持菜鸟教程www.piaodoo.com。

[Link 1]: https://www.piaodoo.com/
[Link 2]: https://www.hnthzk.com/
[Link 3]: http://www.op-kg.com/