Understanding of PointNet network architecture

Words= 1.4k   |   Time≈ 9min

1. Architecture

Pointnet

2.Feature Transformation Networks T-Net

PointNet solves two key problems:

  • the invariance of point cloud transformation
  • the disorder of point cloud.

For the invariance of point cloud transformation, the class of the point cloud object will not change after rotation, PointNet refers to the STN in 2D deep learning on this issue, and adds T-Net Network architecture here to spatially transform the input point cloud, making it as invariant to rotation as possible.

feat_transform.py:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# -*- coding: utf-8 -*-
# Implementation of the Feature Transformation Networks T-Net
import os
import sys
import numpy as np
import torch
import functools
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torchsummary import summary

'''
[description]
input_transform_net
Input (XYZ) Transform Net, input is Bx3xN
B->Batch size
N->number of points per batch
        Return:
            Transformation matrix of size 3x3
'''
class input_transform_net(nn.Module):
    def __init__(self):
        super(input_transform_net, self).__init__()
        self.conv1 = torch.nn.Conv1d(3, 64, 1)
        self.bn1 = nn.BatchNorm1d(64)

        self.conv2 = torch.nn.Conv1d(64, 128, 1)
        self.bn2 = nn.BatchNorm1d(128)

        self.conv3 = torch.nn.Conv1d(128, 1024, 1)
        self.bn3 = nn.BatchNorm1d(1024)

        self.fc1 = nn.Linear(1024, 512)
        self.fc2 = nn.Linear(512, 256)
        self.fc3 = nn.Linear(256, 9)
        self.relu = nn.ReLU()

        self.bn4 = nn.BatchNorm1d(512)
        self.bn5 = nn.BatchNorm1d(256)

    def forward(self, x):
        batchsize = x.size()[0]
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))

        # The number of channels has been adjusted to 1024
        # Use a maxpooling for every sample point set 
        x = torch.max(x, 2, keepdim=True)[0]
        x = x.view(-1, 1024)

        x = F.relu(self.bn4(self.fc1(x)))
        x = F.relu(self.bn5(self.fc2(x)))
        x = self.fc3(x)

        # Use an Identity matrix to do initialization
        iden = Variable(torch.from_numpy
          (np.array([1, 0, 0, 0, 1, 0, 0, 0,1])
           .astype(np.float32))).view(1, 9)
           .repeat(batchsize, 1)
        if x.is_cuda: #Use GPU to accelerate calculations
            iden = iden.cuda()
        x = x + iden
        x = x.view(-1, 3, 3)
        return x


'''
[description]
feature_transform_net
Feature Transform Net, input is BxKxN
        Return:
            Transformation matrix of size KxK
'''
class feature_transform_net(nn.Module):
    def __init__(self, K=64):
        super(feature_transform_net, self).__init__()
        self.K = K

        self.conv1 = torch.nn.Conv1d(64, 64, 1)
        self.bn1 = nn.BatchNorm1d(64)

        self.conv2 = torch.nn.Conv1d(64, 128, 1)
        self.bn2 = nn.BatchNorm1d(128)

        self.conv3 = torch.nn.Conv1d(128, 1024, 1)
        self.bn3 = nn.BatchNorm1d(1024)

        self.fc1 = nn.Linear(1024, 512)
        self.bn4 = nn.BatchNorm1d(512)

        self.fc2 = nn.Linear(512, 256)
        self.bn5 = nn.BatchNorm1d(256)

        self.fc3 = nn.Linear(256, self.K**2)
        self.relu = nn.ReLU()

    def forward(self, x):
        batchsize = x.size()[0]
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))

        # The number of channel has been modified to be 1024
        # 
        # maxpooling for every sample point set 
        x = torch.max(x, 2, keepdim=True)[0]
        x = x.view(-1, 1024)

        x = F.relu(self.bn4(self.fc1(x)))
        x = F.relu(self.bn5(self.fc2(x)))

        x = self.fc3(x)

        iden = Variable(torch.from_numpy(np.eye(self.K)
               .flatten().astype(np.float32)))
               .view(1, self.K**2)
               .repeat(batchsize, 1)
        if x.is_cuda:
            iden = iden.cuda()#Use GPU to accelerate calculations
        x = x + iden
        x = x.view(-1, self.K, self.K)
        return x

3. Implementation of classification and segmentation

The important point is to ensure that the network is invariant to different input orders of the same point cloud. So the PointNet uses a symmetric function. In the implementation of pointnet network, the max-pooling strategy is adopted to solve the disorder problem of point cloud.

Symmetric

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
# -*- coding: utf-8 -*-
import os
import sys
import numpy as np
import torch
import functools
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torchsummary import summary
from feat_transform import *

'''
[description]
PointNetClassify
PointNetSegmentation
input is Bx3xN
        Return:
            BxNxK (K is the number of classes)
'''
class PointNetClassify(nn.Module):
    def __init__(self, class_num=2):
        super(PointNetClassify, self).__init__()
        self.class_num = class_num
        self.input_transform = input_transform_net()

        self.conv1 = torch.nn.Conv1d(3, 64, 1)
        self.bn1 = nn.BatchNorm1d(64)

        self.conv2 = torch.nn.Conv1d(64, 64, 1)
        self.bn2 = nn.BatchNorm1d(64)

        self.feature_transform = feature_transform_net(K=64)

        self.conv3 = torch.nn.Conv1d(64, 64, 1)
        self.bn3 = nn.BatchNorm1d(64)

        self.conv4 = torch.nn.Conv1d(64, 128, 1)
        self.bn4 = nn.BatchNorm1d(128)

        self.conv5 = torch.nn.Conv1d(128, 1024, 1)
        self.bn5 = nn.BatchNorm1d(1024)

        self.fc1 = nn.Linear(1024, 512)
        self.fc_bn1 = nn.BatchNorm1d(512)

        self.fc2 = nn.Linear(512, 256)
        self.fc_bn2 = nn.BatchNorm1d(256)

        self.dropout = nn.Dropout(0.7)

        self.fc3 = nn.Linear(256, self.class_num)

        self.classifier = torch.nn.Conv1d(128, self.class_num, 1)

    def forward(self, x):
        batchsize = x.size()[0]
        n_pts = x.size()[2]  # number of points per sample

        input_transform = self.input_transform(x)
        x = x.transpose(2, 1)  # Bx3xN -> BxNx3
        x = torch.bmm(x, input_transform)  # bmm: batch matrix multiply; x -> BxNx3
        x = x.transpose(2, 1)  # BxNx3 -> Bx3xN

        # mlp(64, 64)
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))     # -> Bx64xN

        feature_transform = self.feature_transform(x)
        x = x.transpose(2, 1)  # Bx64xN -> BxNx64
        x = torch.bmm(x, feature_transform)  # bmm: batch matrix multiply; x -> BxNx64
        x = x.transpose(2, 1)  # BxNx64 -> Bx64xN

        # mlp(64, 128, 1024)
        x = F.relu(self.bn3(self.conv3(x)))
        x = F.relu(self.bn4(self.conv4(x)))
        x = F.relu(self.bn5(self.conv5(x)))     # -> Bx1024xN

        x = torch.max(x, 2, keepdim=True)[0]
        global_feat = x.view(-1, 1024)  # global features, Bx1024

        x = F.relu(self.fc_bn1(self.fc1(global_feat)))
        x = F.relu(self.fc_bn2(self.fc2(x)))
        x = self.dropout(x)
        x = self.fc3(x)

        x = F.log_softmax(x, dim=1)
        return x

'''
[description]
PointNetSegmentation
input is Bx3xN
        Return:
            BxNxK (K is the class num)
'''
class PointNetSegmentation(nn.Module):
    def __init__(self, class_num=2, global_feat=False):
        super(PointNetSegmentation, self).__init__()
        self.class_num = class_num
        self.global_feat = global_feat

        self.input_transform = input_transform_net()

        self.conv1 = torch.nn.Conv1d(3, 64, 1)
        self.bn1 = nn.BatchNorm1d(64)

        self.conv2 = torch.nn.Conv1d(64, 64, 1)
        self.bn2 = nn.BatchNorm1d(64)

        self.feature_transform = feature_transform_net(K=64)

        self.conv3 = torch.nn.Conv1d(64, 64, 1)
        self.bn3 = nn.BatchNorm1d(64)

        self.conv4 = torch.nn.Conv1d(64, 128, 1)
        self.bn4 = nn.BatchNorm1d(128)

        self.conv5 = torch.nn.Conv1d(128, 1024, 1)
        self.bn5 = nn.BatchNorm1d(1024)

        # segmentation network
        if not self.global_feat:
            self.conv6 = torch.nn.Conv1d(1088, 512, 1)
            self.bn6 = nn.BatchNorm1d(512)
        else:
            self.conv6 = torch.nn.Conv1d(1024, 512, 1)
            self.bn6 = nn.BatchNorm1d(512)

        self.conv7 = torch.nn.Conv1d(512, 256, 1)
        self.bn7 = nn.BatchNorm1d(256)

        self.conv8 = torch.nn.Conv1d(256, 128, 1)
        self.bn8 = nn.BatchNorm1d(128)

        self.classifier = torch.nn.Conv1d(128, self.class_num, 1)

    def forward(self, x):
        batchsize = x.size()[0]
        n_pts = x.size()[2]  # number of points per sample

        input_transform = self.input_transform(x)
        x = x.transpose(2, 1)  # Bx3xN -> BxNx3
        x = torch.bmm(x, input_transform)  # bmm: batch matrix multiply; x -> BxNx3
        x = x.transpose(2, 1)  # BxNx3 -> Bx3xN

        # mlp(64, 64)
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))     # -> Bx64xN

        feature_transform = self.feature_transform(x)
        x = x.transpose(2, 1)  # Bx64xN -> BxNx64
        x = torch.bmm(x, feature_transform)  # bmm: batch matrix multiply; x -> BxNx64
        x = x.transpose(2, 1)  # BxNx64 -> Bx64xN

        # to save memory
        del feature_transform
        del input_transform

        # Feature for per point, local feature
        pointfeat = x

        # mlp(64, 128, 1024)
        x = F.relu(self.bn3(self.conv3(x)))
        x = F.relu(self.bn4(self.conv4(x)))
        x = F.relu(self.bn5(self.conv5(x)))     # -> Bx1024xN

        x = torch.max(x, 2, keepdim=True)[0]
        x = x.view(-1, 1024)  # global features, Bx1024

        x = x.view(-1, 1024, 1).repeat(1, 1, n_pts)  # Bx1024 -> Bx1024xN

        # Global-feature + point-feature
        if not self.global_feat:
            x =  torch.cat([x, pointfeat], 1)

        # mlp(512, 256, 128)
        x = F.relu(self.bn6(self.conv6(x)))
        x = F.relu(self.bn7(self.conv7(x)))
        x = F.relu(self.bn8(self.conv8(x)))

        # mlp(128, m)
        x = self.classifier(x)

        x = x.transpose(2, 1).contiguous()

        x = F.log_softmax(x.view(-1, self.class_num), dim=-1)
        x = x.view(batchsize, n_pts, self.class_num)
        return x

if __name__ == '__main__':
    sim_data = Variable(torch.rand(32,3,2500))

    cly = PointNetClassify(class_num = 40)
    out = cly(sim_data)
    print('cly', out.size())

    seg = PointNetSegmentation(class_num = 40)
    out = seg(sim_data)
    print('seg', out.size())

扫一扫，分享到微信

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

研究方向：
计算机视觉，图像深度学习
邮箱：yansz34@qq.com
期待讨论！