Python计算机视觉-第10章

本章概述如何通过 Python 接口使用流行的计算机视觉库 OpenCV。OpenCV 是一个 C++ 库，用于(实时)处理计算视觉问题。实时处理计算机视觉的 C++ 库，最初由英特尔公司开发，现由 Willow Garage 维护。OpenCV 是在 BSD 许可下发布的开源库，这意味着它对于学术研究和商业应用是免费的。OpenCV 2.0 版本对于 Python 的支持已经得到了极大的改善。下面，我们会讲解一些基本的例子并深入了解视频与跟踪。

1、OpenCV基础

（1）图像的读写

# -*- coding: utf-8 -*-
import cv2

# 读入图像
im = cv2.imread('../data/empire.jpg')

# 打印图像尺寸
h, w = im.shape[:2]
print (h, w)

# 保存原jpg格式的图像为png格式图像
cv2.imwrite('../images/ch10/ch10_P228_Reading-and-Writing-Images.png',im)

（2）计算积分图像

# -*- coding: utf-8 -*-
import cv2
from pylab import *


# 添加中文字体支持
from matplotlib.font_manager import FontProperties
font = FontProperties(fname=r"/System/Library/Fonts/PingFang.ttc", size=14)

# 读入图像
im = cv2.imread('../data/fisherman.jpg')
# 转换颜色空间
gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)

# 显示积分图像
fig = plt.figure()
subplot(121)
plt.gray()
imshow(gray)
title(u'灰度图', fontproperties=font)
axis('off')

# 计算积分图像
intim = cv2.integral(gray)
# 归一化
intim = (255.0*intim) / intim.max()

#显示积分图像
subplot(122)
plt.gray()
imshow(intim)
title(u'积分图', fontproperties=font)
axis('off')
show()

# 用OpenCV显示图像
cv2.imshow("Image", intim)
cv2.waitKey()

# 用OpenCV保存积分图像
cv2.imwrite('../images/ch10/ch10_P230_Displaying-Images-and-Results-cv2.jpg',intim)

# 保存figure中的灰度图像和积分图像
fig.savefig("../images/ch10/ch10_P230_Displaying-Images-and-Results.png")

（3）泛洪填充

# -*- coding: utf-8 -*-
import cv2
import numpy
from pylab import *


# 添加中文字体支持
from matplotlib.font_manager import FontProperties
font = FontProperties(fname="/System/Library/Fonts/PingFang.ttc", size=14)

# 读入图像
filename = '../data/fisherman.jpg'
im = cv2.imread(filename)
# 转换颜色空间
rgbIm = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)

# 显示原图
fig = plt.figure()
subplot(121)
plt.gray()
imshow(rgbIm)
title(u'原图', fontproperties=font)
axis('off')

# 获取图像尺寸
h, w = im.shape[:2]
# 泛洪填充
diff = (6, 6, 6)
mask = zeros((h+2, w+2), numpy.uint8)
cv2.floodFill(im, mask, (10, 10), (255, 255, 0), diff, diff)

# 显示泛洪填充后的结果
subplot(122)
imshow(im)
title(u'泛洪填充', fontproperties=font)
axis('off')

show()
fig.savefig("../images/ch10/floodFill.png")

# 在OpenCV窗口中显示泛洪填充后的结果
cv2.imshow('flood fill', im)
cv2.waitKey()
# 保存结果
cv2.imwrite('../images/ch10/floodFill.jpg',im)

（4）提取 SURF 特征

# -*- coding: utf-8 -*-
import cv2
import numpy
from pylab import *


# 读入图像
im = cv2.imread('../data/empire.jpg')
# 下采样
im_lowres = cv2.pyrDown(im)
# 转化为灰度图像
gray = cv2.cvtColor(im_lowres, cv2.COLOR_RGB2GRAY)
# 检测特征点
# s = cv2.SURF()   ##AttributeError: module 'cv2.cv2' has no attribute 'SURF'
s = cv2.xfeatures2d.SURF_create()  ##卸载高版本opencv-pyhton和opencv-contribute-python安装3.4.2.16版本
mask = numpy.uint8(ones(gray.shape))
keypoints = s.detect(gray, mask)
# 显示图像及特征点
vis = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
for k in keypoints[::10]:  ##每隔10个像素取1个
    cv2.circle(vis, (int(k.pt[0]), int(k.pt[1])), 2, (0, 255, 0), -1)
    cv2.circle(vis, (int(k.pt[0]), int(k.pt[1])), int(k.size), (0, 255, 0), 2)
cv2.imshow('local descriptors', vis)
cv2.waitKey()

cv2.imwrite('../images/ch10/ch10_P232_Fig10-3.jpg',vis)

2、处理视频

（1）视频输入

import cv2

# setup video capture
cap = cv2.VideoCapture(0)
cap.open(0)
while True:
    ret,im = cap.read()
    cv2.imshow('video test',im)
    key = cv2.waitKey(10)
    if key == 27:
        break

（2）视频模糊

import cv2

# setup video capture
cap = cv2.VideoCapture(0)
cap.open(0)
# get frame, apply Gaussian smoothing, show result
while True:
    ret, im = cap.read()
    blur = cv2.GaussianBlur(im, (0, 0), 5)
    cv2.imshow('camera blur', blur)
    if cv2.waitKey(10) == 27:
      break

(3)视频读取

from scipy import *
import cv2

# setup video capture
cap = cv2.VideoCapture(0)
cap.open(0)
if cap.isOpened():
  print("Finally")
else:
  print("BOOM")
frames = []
# get frame, store in array
while True:
    ret, im = cap.read()
    cv2.imshow('video', im)
    frames.append(im)
    if cv2.waitKey(10) == 27:
      break
frames = array(frames)
# check the sizes
print(im.shape)
print(frames.shape)

3、视频追踪

（1）光流追踪

from scipy import *  ##NameError: name 'mgrid' is not defined，加入scipy库或者numpy库
import cv2

def draw_flow(im,flow,step=16):
    """ Plot optical flow at sample points
    spaced step pixels apart. """
    h,w = im.shape[:2]
    y,x = mgrid[step/2:h:step,step/2:w:step].reshape(2,-1).astype(int)  ##fx,fy = flow[y,x].T IndexError: arrays used as indices must be of integer (or boolean) type，修改.astype(int)
    fx,fy = flow[y,x].T
    # create line endpoints
    lines = vstack([x,y,x+fx,y+fy]).T.reshape(-1,2,2)
    lines = int32(lines)
    # create image and draw
    vis = cv2.cvtColor(im,cv2.COLOR_GRAY2BGR)
    for (x1,y1),(x2,y2) in lines:
        cv2.line(vis,(x1,y1),(x2,y2),(0,255,0),1)
        cv2.circle(vis,(x1,y1),1,(0,255,0), -1)
    return vis

# setup video capture
cap = cv2.VideoCapture(0)
ret,im = cap.read()
prev_gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
while True:
    # get grayscale image
    ret,im = cap.read()
    gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
    # compute flow
    flow = cv2.calcOpticalFlowFarneback(prev_gray,gray,None,0.5,3,15,3,5,1.2,0)
    prev_gray = gray
    # plot the flow vectors
    cv2.imshow('Optical flow',draw_flow(gray,flow))
    if cv2.waitKey(10) == 27:
        break

（2）Lucas-Kanade算法跟踪器

lktrack.py

from numpy import *
import cv2


# some constants and default parameters
lk_params = dict(winSize=(15,15),maxLevel=2,
                criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT,10,0.03)) 

subpix_params = dict(zeroZone=(-1,-1),winSize=(10,10),
                     criteria = (cv2.TERM_CRITERIA_COUNT | cv2.TERM_CRITERIA_EPS,20,0.03))

feature_params = dict(maxCorners=500,qualityLevel=0.01,minDistance=10)


class LKTracker(object):
    """    Class for Lucas-Kanade tracking with 
        pyramidal optical flow."""

    def __init__(self,imnames):
        """    Initialize with a list of image names. """

        self.imnames = imnames
        self.features = []
        self.tracks = []
        self.current_frame = 0

    def step(self,framenbr=None):
        """    Step to another frame. If no argument is 
            given, step to the next frame. """

        if framenbr is None:
            self.current_frame = (self.current_frame + 1) % len(self.imnames)
        else:
            self.current_frame = framenbr % len(self.imnames)

    def detect_points(self):
        """    Detect 'good features to track' (corners) in the current frame
            using sub-pixel accuracy. """

        # load the image and create grayscale
        self.image = cv2.imread(self.imnames[self.current_frame])
        self.gray = cv2.cvtColor(self.image,cv2.COLOR_BGR2GRAY)

        # search for good points
        features = cv2.goodFeaturesToTrack(self.gray, **feature_params)

        # refine the corner locations
        cv2.cornerSubPix(self.gray,features, **subpix_params)
        
        self.features = features
        self.tracks = [[p] for p in features.reshape((-1,2))]
        
        self.prev_gray = self.gray

    def track_points(self):
        """    Track the detected features. """

        if self.features != []:
            self.step() # move to the next frame
            
            # load the image and create grayscale
            self.image = cv2.imread(self.imnames[self.current_frame])
            self.gray = cv2.cvtColor(self.image,cv2.COLOR_BGR2GRAY)
            
            # reshape to fit input format
            tmp = float32(self.features).reshape(-1, 1, 2)
            
            # calculate optical flow
            features,status,track_error = cv2.calcOpticalFlowPyrLK(self.prev_gray,self.gray,tmp,None,**lk_params)

            # remove points lost
            self.features = [p for (st,p) in zip(status,features) if st]
            
            # clean tracks from lost points
            features = array(features).reshape((-1,2))
            for i,f in enumerate(features):
                self.tracks[i].append(f)
            ndx = [i for (i,st) in enumerate(status) if not st]
            ndx.reverse() #remove from back
            for i in ndx:
                self.tracks.pop(i)
            
            self.prev_gray = self.gray
            
    def track(self):
        """    Generator for stepping through a sequence."""

        for i in range(len(self.imnames)):
            if self.features == []:
                self.detect_points()
            else:
                self.track_points()
            
            # create a copy in RGB
            f = array(self.features).reshape(-1,2)
            im = cv2.cvtColor(self.image,cv2.COLOR_BGR2RGB)
            yield im,f

    def draw(self):
        """    Draw the current image with points using
            OpenCV's own drawing functions. 
            Press ant key to close window."""

        # draw points as green circles
        for point in self.features:
            cv2.circle(self.image,(int(point[0][0]),int(point[0][1])),3,(0,255,0),-1)
        
        cv2.imshow('LKtrack',self.image)
        cv2.waitKey()

跟踪器

import lktrack

imnames = ['../data/bt/bt.003.pgm', '../data/bt/bt.002.pgm', '../data/bt/bt.001.pgm', '../data/bt/bt.000.pgm']
# create tracker object
lkt = lktrack.LKTracker(imnames)
# detect in first frame, track in the remaining
lkt.detect_points()
lkt.draw()
for i in range(len(imnames)-1):
    lkt.track_points()
    lkt.draw()

(3)Lucas-Kanade算法发生器

发生器

import lktrack
from pylab import *
imnames = ['../data/viff/viff.000.ppm', '../data/viff/viff.001.ppm',
           '../data/viff/viff.002.ppm', '../data/viff/viff.003.ppm', '../data/viff/viff.004.ppm']
# track using the LKTracker generator
lkt = lktrack.LKTracker(imnames)
for im,ft in lkt.track():
    print('tracking %d features' % len(ft))

# plot the tracks
figure()
imshow(im)
for p in ft:
    plot(p[0],p[1],'bo')
for t in lkt.tracks:
    plot([p[0] for p in t],[p[1] for p in t])
axis('off')
show()

4、OpenCV算子

如下图所示，OpenCV中有60多个图像视觉算子涉及图像图像分割、检测、识别、追踪各个方面，其中calibrate.py是矫正脚本，demo.py是算法集合脚本。使用方法 $ python xxx.py 或 $ python xxx.py yyy.jpg，e.g.

$ python inpaint.py empire.jpg

$ python watershed.py empire.jpg

$ python houghlines.py empire.jpg

如上图所示，OpenCV算子源码都在/usr/local/Cellar/opencv-3.3.0/samples/python/文件夹下。以tst_scene_render.py算子为例调用如下

AbnerdeMacBook-Pro:~ Abner$ cd /usr/local/Cellar/opencv-3.3.0/samples/python/

AbnerdeMacBook-Pro:python Abner$ python tst_scene_render.py

5、其他

（1）开发环境

本代码是在mac电脑sublimetxt编辑器python3.7.8下调试出来的，如果你要在windows/linux下编辑器/IDE的其他python版本运行的话，请对代码做相应的调整。

(2) opencv版本

1）s = cv2.SURF() 报错 AttributeError: module 'cv2.cv2' has no attribute 'SURF'。这是因为早期SURF函数是在cv2库直接调用的，后期cv2将SURF、SIFT、Harris等算子整合到opencv-contrib-python这个包里了，所以不再能直接调用。SURF、SIFT、Harris等算子被封装在cv2.xfeatures2d中了，安装opencv-contrib-python这个包后，调用cv2.xfeatures2d.SURF_create()就可以替代早期版本中的cv2.SURF()。

2）s = cv2.xfeatures2d.SURF_create() 报错AttributeError: module 'cv2' has no attribute 'xfeatures2d'。这是因为SURF、SIFT、Harris算子都已经申请专利了，所以这些算子不免费，在opencv3.4.2之后的版本都不再包含这两个算子。如1）所述，opencv3.4.2之前的版本将SURF、SIFT、Harris等算子整合到opencv-contrib-python里了，做一些特征检测匹配，所以需要用到opencv-contrib-python这个包。综合考虑我使用3.4.2.16版本的opencv-python和opencv-contrib-python。

（3）源码和图片

已经调试过的源码和图片详见：

https://github.com/Abonaventure/pcv-book-code.git

或

https://gitlab.com/Abonaventure/pcv-book-code.git