Creates a 3D visualization using matplotlib
def render_animation(
keypoints, keypoints_metadata, poses, skeleton, fps, bitrate,
azim, output, viewport, limit=-1, downsample=1,
size=6, input_video_path=None, input_video_skip=0):
"""
TODO
Render an animation. The supported output modes are:
-- 'interactive': display an interactive figure
(also works on notebooks if associated with %matplotlib inline)
-- 'html': render the animation as HTML5 video. Can be displayed in
a notebook using HTML(...).
-- 'filename.mp4': render and export the animation as an h264 video
(requires ffmpeg).
-- 'filename.gif': render and export the animation a gif file
(requires imagemagick).
"""
# Initializes the input plot (2D)
plt.ioff()
fig = plt.figure(figsize=(size*(1 + len(poses)), size))
ax_in = fig.add_subplot(1, 1 + len(poses), 1)
ax_in.get_xaxis().set_visible(False)
ax_in.get_yaxis().set_visible(False)
ax_in.set_axis_off()
ax_in.set_title('Input')
# This handles the 3D plot:
ax_3d = []
lines_3d = []
trajectories = []
radius = 1.7
for index, (title, data) in enumerate(poses.items()):
ax = fig.add_subplot(1, 1 + len(poses), index+2, projection='3d')
ax.view_init(elev=15., azim=azim)
ax.set_xlim3d([-radius/2, radius/2])
ax.set_zlim3d([0, radius])
ax.set_ylim3d([-radius/2, radius/2])
try:
ax.set_aspect('equal')
except NotImplementedError:
ax.set_aspect('auto')
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.set_zticklabels([])
ax.dist = 7.5
ax.set_title(title) #, pad=35
ax_3d.append(ax)
lines_3d.append([])
trajectories.append(data[:, 0, [0, 1]])
poses = list(poses.values())
# Decode video i.e the left side 2D plot
if input_video_path is None:
# Black background
all_frames = np.zeros(
(keypoints.shape[0], viewport[1], viewport[0]), dtype='uint8'
)
else:
# Load video using ffmpeg
all_frames = []
for f in read_video(input_video_path, skip=input_video_skip, limit=limit):
all_frames.append(f)
effective_length = min(keypoints.shape[0], len(all_frames))
all_frames = all_frames[:effective_length]
keypoints = keypoints[input_video_skip:] # todo remove
for idx in range(len(poses)):
poses[idx] = poses[idx][input_video_skip:]
if fps is None:
fps = get_fps(input_video_path)
if downsample > 1:
keypoints = downsample_tensor(keypoints, downsample)
all_frames = downsample_tensor(
np.array(all_frames), downsample).astype('uint8')
for idx in range(len(poses)):
poses[idx] = downsample_tensor(poses[idx], downsample)
trajectories[idx] = downsample_tensor(trajectories[idx], downsample)
fps /= downsample
initialized = False
image = None
lines = []
points = None
if limit < 1:
limit = len(all_frames)
else:
limit = min(limit, len(all_frames))
parents = skeleton.parents()
# Holds all the associated values to each joint in the skeleton
# model that is needed to calculate a specific angle.
joints = {
"LElbow": [3, 10, 12],
"RElbow": [4, 11, 13],
"Neck": [0, 1, 2],
"Spine": [1, 2, 3],
"LKnee": [6, 14, 16],
"RKnee": [8, 15, 17],
"LFot": [14, 16, 7],
"RFot": [15, 17, 9]
}
# Instansiate a dictionary that holds all angle calculations for the joints
angles = {}
def update_video(i):
"""Updates the video with a new frame, used in FuncAnim."""
nonlocal initialized, image, lines, points
for n, ax in enumerate(ax_3d):
ax.set_xlim3d(
[-radius/2 + trajectories[n][i, 0], radius/2
+ trajectories[n][i, 0]]
)
ax.set_ylim3d(
[-radius/2 + trajectories[n][i, 1], radius/2
+ trajectories[n][i, 1]]
)
# Update 2D poses
joints_right_2d = keypoints_metadata['keypoints_symmetry'][1]
colors_2d = np.full(keypoints.shape[1], 'black')
colors_2d[joints_right_2d] = 'red'
# This runs the first time we render animation.
if not initialized:
image = ax_in.imshow(all_frames[i], aspect='equal')
# NOTE: pos is (num_joints_out, dim) for each frame
pos = 0
for j, j_parent in enumerate(parents):
if j_parent == -1:
continue
if len(parents) == keypoints.shape[1] \
and keypoints_metadata['layout_name'] != 'coco':
# Draw skeleton only if keypoints match (otherwise
# we don't have the parents definition)
lines.append(ax_in.plot(
[keypoints[i, j, 0], keypoints[i, j_parent, 0]],
[keypoints[i, j, 1], keypoints[i, j_parent, 1]],
color='pink'
)
)
# Color of the skelton
col = 'red' if j in skeleton.joints_right() else 'black'
for n, ax in enumerate(ax_3d):
pos = poses[n][i]
lines_3d[n].append(ax.plot(
[pos[j, 0], pos[j_parent, 0]],
[pos[j, 1], pos[j_parent, 1]],
[pos[j, 2], pos[j_parent, 2]], zdir='z', c=col)
)
# Angle calculation
for joint in joints.keys():
keyp = joints[joint]
angle = angle_calc(
pos[keyp[0], :],
pos[keyp[1], :],
pos[keyp[2], :]
)
angles[joint] = [angle]
points = ax_in.scatter(
*keypoints[i].T, 10, color=colors_2d,
edgecolors='white', zorder=10
)
initialized = True
else:
image.set_data(all_frames[i])
# NOTE: pos is (num_joints_out, dim) for each frame
pos = 0
for j, j_parent in enumerate(parents):
if j_parent == -1:
continue
if len(parents) == keypoints.shape[1] and \
keypoints_metadata['layout_name'] != 'coco':
lines[j-1][0].set_data(
[keypoints[i, j, 0], keypoints[i, j_parent, 0]],
[keypoints[i, j, 1], keypoints[i, j_parent, 1]]
)
for n, ax in enumerate(ax_3d):
pos = poses[n][i]
lines_3d[n][j-1][0].set_xdata(
np.array([pos[j, 0], pos[j_parent, 0]]))
lines_3d[n][j-1][0].set_ydata(
np.array([pos[j, 1], pos[j_parent, 1]]))
lines_3d[n][j-1][0].set_3d_properties(
np.array([pos[j, 2], pos[j_parent, 2]]), zdir='z')
# Angle calculation
for joint in joints.keys():
keyp = joints[joint]
angle = angle_calc(
pos[keyp[0], :],
pos[keyp[1], :],
pos[keyp[2], :]
)
angles[joint].append(angle)
points.set_offsets(keypoints[i])
print('{}/{}'.format(i, limit), end='\r')
fig.tight_layout()
anim = FuncAnimation(
fig, update_video, frames=np.arange(0, limit),
interval=1000/fps, repeat=False)
if output.endswith('.mp4'):
Writer = writers['ffmpeg']
writer = Writer(fps=fps, metadata={}, bitrate=bitrate)
anim.save(output, writer=writer)
elif output.endswith('.gif'):
anim.save(output, dpi=80, writer='pillow')
else:
raise ValueError(
'Unsupported output format (only .mp4 and .gif are supported)')
plt.close()
# Save angles as a json file for web app to read.
with open('angles.json', 'w', encoding='utf-8') as f:
json.dump(angles, f, ensure_ascii=False, indent=4)
f.close()