torchnet+VGG16计算patch之间相似度

torchnet+VGG16计算patch之间相似度

torch

VGG16

similarity

本来打算使用VGG实现siamese CNN的，但是没想明白怎么使用torchnet对模型进行微调。。。所以只好把VGG的卷积层单独做一个数据预处理模块，后面跟一个网络，将两个VGG输出的结果输入该网络中，仅训练这个浅层网络。

数据：使用了MOTChallenge数据库MOT16-02中的pedestrian

代码：

1. -- --------------------------------------------------------------------------------------- 

   ---

2. -- 读取MOT16-02数据集的groundtruth，分成训练集和测试集 

   ---

3. -- --------------------------------------------------------------------------------------- 

   ---

4. require 'torch' 

   ---

5. require 'cutorch' 

   ---

6. torch.setdefaulttensortype('torch.FloatTensor') 

   ---

7. data_type = 'torch.CudaTensor' -- 设置数据类型，不适用GPU可以设置为torch.FloatTensor 

   ---

8.  

   ---

9. require 'image' 

   ---

10. local datapath = '/home/zwzhou/programFiles/2DMOT2015/MOT16/train/MOT16-02/' 

    ---

11. local tmp = image.load(datapath .. 'img1/000001.jpg',3,'byte') 

    ---

12. local width = tmp:size(3) 

    ---

13. local height = tmp:size(2) 

    ---

14. local num = 600 

    ---

15. local imgs = torch.Tensor(num,3,height,width) 

    ---

16.  

    ---

17. local file,_ = io.open('imgs.t7') 

    ---

18. if not file then 

    ---

19. for i=1,num do -- 读取视频帧 

    ---

20. imgs[i]=image.load(datapath .. 'img1/' .. string.format('%06d.jpg',i)) 

    ---

21. end 

    ---

22. torch.save('imgs.t7',imgs) 

    ---

23. else 

    ---

24. imgs = torch.load('imgs.t7') 

    ---

25. end 

    ---

26.  

    ---

27. require'sys' 

    ---

28. local gt_path = datapath .. 'gt/gt.txt' 

    ---

29. local gt_info={} 

    ---

30. local i=0 

    ---

31. for line in io.lines(gt_path) do -- pedestrians的patch信息 

    ---

32. local v=sys.split(line,',') 

    ---

33. if tonumber(v[7]) ==1 and tonumber(v[9]) > 0.8 then -- 筛选有效的patch，是pedestrian且可见度>0.8 

    ---

34. table.insert(gt_info,{tonumber(v[1]),tonumber(v[2]),tonumber(v[3]),tonumber(v[4]),tonumber(v[5]),tonumber(v[6])}) 

    ---

35. -- 对应的是frame index,track index, x, y, w, h 

    ---

36. end 

    ---

37. end 

    ---

38. -- 构建样本对，这里主要是为了正负样本个数相同，每个pedestrian选取25个相同id的patch，25个不同id的patch 

    ---

39. local pairwise={} 

    ---

40. for i=1,#gt_info do 

    ---

41. local count=0 

    ---

42. local iter=0 

    ---

43. repeat  

    ---

44. local j=torch.ceil(torch.rand(1)*(#gt_info))[1] 

    ---

45. if gt_info[i][2] == gt_info[j][2] then  

    ---

46. count=count+1 

    ---

47. table.insert(pairwise,{i,j}) 

    ---

48. end 

    ---

49. iter=iter+1 

    ---

50. until(count >25 or iter>100) 

    ---

51. repeat  

    ---

52. local j=torch.ceil(torch.rand(1)*#gt_info)[1] 

    ---

53. if gt_info[i][2] ~= gt_info[j][2] then  

    ---

54. count=count-1 

    ---

55. table.insert(pairwise,{i,j}) 

    ---

56. end 

    ---

57. until(count <0) 

    ---

58. end 

    ---

59.  

    ---

60. local function cast(x) return x:type(data_type) end -- 类型转换 

    ---

61.  

    ---

62. -- 加载pretrained VGG16 model 

    ---

63. require 'nn' 

    ---

64. require 'loadcaffe' 

    ---

65. local function getPretrainedModel() 

    ---

66. local proto = '/home/zwzhou/modelZoo/VGG_ILSVRC_16_layers_deploy.prototxt' 

    ---

67. local caffemodel = '/home/zwzhou/modelZoo/VGG_ILSVRC_16_layers.caffemodel' 

    ---

68. local VGG16 = loadcaffe.load(proto,caffemodel,'nn') 

    ---

69. for i = 1,3 do 

    ---

70. VGG16.modules[#VGG16.modules]=nil 

    ---

71. end 

    ---

72. return VGG16 

    ---

73. end 

    ---

74.  

    ---

75. -- 为了能够使用VGG，需要定义一些预处理方法 

    ---

76. local loadSize = {3,256,256} 

    ---

77. local sampleSize={3,224,224} 

    ---

78.  

    ---

79. local function adjustScale(input) -- VGG需要先将输入图片的最小边缩放到256，另一边保持纵横比 

    ---

80. if input:size(3) < input:size(2) then 

    ---

81. input = image.scale(input,loadSize[2],loadSize[3]*input:size(2)/input:size(3)) 

    ---

82. else 

    ---

83. input = image.scale(input,loadSize[2]*input:size(3)/input:size(2),loadSize[3]) 

    ---

84. end 

    ---

85. return input 

    ---

86. end 

    ---

87.  

    ---

88. local bgr_means = {103.939,116.779,123.68} -- VGG使用的均值，注意是BGR通道，image.load()获得的是rgb 

    ---

89. local function vggProcessing(img) 

    ---

90. local img2 = img:clone() -- 深度拷贝 

    ---

91. img2[{{1}}] = img[{{3}}] 

    ---

92. img2[{{3}}] = img[{{1}}] -- rgb -> bgr 

    ---

93. img2=img2:mul(255) 

    ---

94. for i=1,3 do 

    ---

95. img2[i]:add(-bgr_means[i]) 

    ---

96. end 

    ---

97. return img2 

    ---

98. end 

    ---

99.  

    ---

100. local function centerCrop(input) -- 截取224*224大小 

     ---

101. local oH = sampleSize[2] 

     ---

102. local oW = sampleSize[3] 

     ---

103. local iW = input:size(3) 

     ---

104. local iH = input:size(2) 

     ---

105. local w1 = math.ceil((iW-oW)/2) 

     ---

106. local h1 = math.ceil((iH-oH)/2) 

     ---

107. local out = image.crop(input,w1,h1,w1+oW,h1+oH) 

     ---

108. return out 

     ---

109. end 

     ---

110.  

     ---

111. local file,_ = io.open('vgg_info.t7') 

     ---

112. local vgg_info={} 

     ---

113. if not file then 

     ---

114. local VGG16_model = getPretrainedModel() 

     ---

115. if data_type:match'torch.Cuda.*Tensor' then 

     ---

116. require 'cudnn' 

     ---

117. require 'cunn' 

     ---

118. cudnn.convert(VGG16_model,cudnn):cuda() 

     ---

119. cudnn.benchmark = true 

     ---

120. end 

     ---

121. cast(VGG16_model) 

     ---

122. for i=1, #gt_info do 

     ---

123. local idx=gt_info[i] 

     ---

124. local img = imgs[idx[1]] 

     ---

125. local x1 = math.max(idx[3],1) 

     ---

126. local y1 = math.max(idx[4],1) 

     ---

127. local x2 = math.min(idx[3]+idx[5],width) 

     ---

128. local y2 = math.min(idx[4]+idx[6],height) 

     ---

129. local patch = image.crop(img,x1,y1,x2,y2) 

     ---

130. patch = adjustScale(patch) 

     ---

131. patch = vggProcessing(patch) 

     ---

132. patch = centerCrop(patch) 

     ---

133. patch=cast(patch) 

     ---

134. table.insert(vgg_info,VGG16_model:forward(patch):float()) 

     ---

135. end 

     ---

136. torch.save('vgg_info.t7',vgg_info) 

     ---

137. else  

     ---

138. vgg_info=torch.load('vgg_info.t7') 

     ---

139. end 

     ---

140.  

     ---

141. local function getPatchPair(tmp) -- 获得patch 对 

     ---

142. local pp = {} 

     ---

143. pp[1] = vgg_info[tmp[1]] 

     ---

144. pp[2] = vgg_info[tmp[2]] 

     ---

145. local t=torch.cat(pp[1],pp[2],1) 

     ---

146. return t 

     ---

147. end 

     ---

148.  

     ---

149. -- 定义datasetiterator 

     ---

150. local tnt=require'torchnet' 

     ---

151. local function getIterator(mode) 

     ---

152. -- 创建model 

     ---

153. local fc = nn.Sequential() 

     ---

154. fc:add(nn.View(-1,4096*2)) 

     ---

155. fc:add(nn.Linear(4096*2,500)) 

     ---

156. fc:add(nn.ReLU(true)) 

     ---

157. fc:add(nn.Normalize(2)) 

     ---

158. fc:add(nn.Linear(500,500)) 

     ---

159. fc:add(nn.ReLU(true)) 

     ---

160. fc:add(nn.Linear(500,1)) 

     ---

161.  

     ---

162. -- print(fc:forward(torch.randn(2,4096*2))) 

     ---

163. if data_type:match'torch.Cuda.*Tensor' then 

     ---

164. require 'cudnn' 

     ---

165. require 'cunn' 

     ---

166. cudnn.convert(fc,cudnn):cuda() 

     ---

167. cudnn.benchmark = true 

     ---

168. end 

     ---

169. cast(fc) 

     ---

170.  

     ---

171. -- 构建训练引擎，使用OptimEngine 

     ---

172. require 'optim' 

     ---

173. local engine = tnt.OptimEngine() 

     ---

174. local criterion = cast(nn.MarginCriterion()) 

     ---

175.  

     ---

176. -- 创建一些评估值 

     ---

177. local train_timer = torch.Timer() 

     ---

178. local test_timer = torch.Timer() 

     ---

179. local data_timer = torch.Timer() 

     ---

180.  

     ---

181. local meter = tnt.AverageValueMeter() -- 用于统计评估函数的输出 

     ---

182. local confusion = optim.ConfusionMatrix(2) -- 2类混淆矩阵 

     ---

183. local data_time_meter = tnt.AverageValueMeter() 

     ---

184. -- log 

     ---

185. local logtext=require 'torchnet.log.view.text' 

     ---

186. log = tnt.Log{ 

     ---

187. keys = {'train_loss','train_acc','data_loading_time','epoch','test_acc','train_time','test_time'}, 

     ---

188. onFlush={ 

     ---

189. logtext{keys={'train_loss','train_acc','data_loading_time','epoch','test_acc','train_time','test_time'}} 

     ---

190. } 

     ---

191. } 

     ---

192.  

     ---

193. local inputs = cast(torch.Tensor()) 

     ---

194. local targets = cast(torch.Tensor()) 

     ---

195.  

     ---

196. -- 填一些hook函数，以便观察训练过程 

     ---

197. engine.hooks.onSample = function(state) 

     ---

198. if state.training then 

     ---

199. data_time_meter:add(data_timer:time().real) 

     ---

200. end 

     ---

201. inputs:resize(state.sample.input:size()):copy(state.sample.input) 

     ---

202. targets:resize(state.sample.target:size()):copy(state.sample.target) 

     ---

203. state.sample.input = inputs 

     ---

204. state.sample.target = targets 

     ---

205. end 

     ---

206.  

     ---

207. engine.hooks.onForwardCriterion = function(state) 

     ---

208. meter:add(state.criterion.output) 

     ---

209. confusion:batchAdd(state.network.output:gt(0):add(1),state.sample.target:gt(0):add(1)) 

     ---

210. end 

     ---

211.  

     ---

212. local function test() -- 用于测试 

     ---

213. engine:test{ 

     ---

214. network = fc, 

     ---

215. iterator = getIterator('test'), 

     ---

216. criterion=criterion,  

     ---

217. } 

     ---

218. confusion:updateValids() 

     ---

219. end 

     ---

220.  

     ---

221. engine.hooks.onStartEpoch = function(state) 

     ---

222. local epoch = state.epoch + 1 

     ---

223. print('===>' .. ' online epoch # ' .. epoch .. '[batchsize = 256]') 

     ---

224. meter:reset() 

     ---

225. confusion:zero() 

     ---

226. train_timer:reset() 

     ---

227. data_time_meter:reset() 

     ---

228. end 

     ---

229.  

     ---

230. engine.hooks.onEndEpoch = function(state) 

     ---

231. local train_loss = meter:value() 

     ---

232. confusion:updateValids() 

     ---

233. local train_acc = confusion.totalValid*100 

     ---

234. local train_time = train_timer:time().real 

     ---

235. meter:reset() 

     ---

236. print(confusion) 

     ---

237. confusion:zero() 

     ---

238. test_timer:reset() 

     ---

239.  

     ---

240. local cache = state.params:clone() -- 保存现场 

     ---

241. --state.params:copy(state.optim.ax) 

     ---

242. test() 

     ---

243. --state.params:copy(cache) -- 恢复现场 

     ---

244.  

     ---

245. log:set{ 

     ---

246. train_loss = train_loss, 

     ---

247. train_acc = train_acc, 

     ---

248. data_loading_time = data_time_meter:value(), 

     ---

249. epoch = state.epoch, 

     ---

250. test_acc = confusion.totalValid*100, 

     ---

251. train_time = train_time, 

     ---

252. test_time = test_timer:time().real, 

     ---

253. } 

     ---

254. log:flush() 

     ---

255. end 

     ---

256.  

     ---

257. engine.hooks.onUpdate = function(state) 

     ---

258. data_timer:reset() 

     ---

259. end 

     ---

260.  

     ---

261. engine:train{ 

     ---

262. network = fc, 

     ---

263. criterion = criterion, 

     ---

264. iterator = getIterator('train'), 

     ---

265. optimMethod = optim.sgd, 

     ---

266. config = {learningRate = 0.05, 

     ---

267. --weightDecay = 0.05, 

     ---

268. momentum = 0.9, 

     ---

269. --t0 = 1e+4, 

     ---

270. --eta0 =0.1 

     ---

271. }, 

     ---

272. maxepoch = 30,  

     ---

273. } 

     ---

274.  

     ---

275. -- 保存模型 

     ---

276. local modelpath = 'SiaVGG16_model.t7' 

     ---

277. print('Saving to ' .. modelpath) 

     ---

278. torch.save(modelpath,fc:float():clearState()) 

     ---

279. --]] 

     ---

输出：

1493386765674.jpg

发现网络太容易过拟合，主要一方面是数据太少，另一方面是视频中就那么几个人，所以patch之间的相关性太大，对网络提供的信息太少。所以使用更多的数据测试结果应该会好许多。

这个代码主要是为了熟悉torchnet package，感受呢，

1. 对于数据的预处理，确实方便多了

2. 如果使用提供的Engine，虽然训练过程简单了但是也太模块化了，比如某些层的微调，比如每层设置不同的学习率

3. 使用Iterator时，尤其要小心