 הצג באתר TensorFlow.org |  הפעל בגוגל קולאב |  הצג ב-GitHub |  הורד מחברת |  ראה דגם TF Hub |
Colab זה מדגים פעולות הכרה בנתוני וידאו באמצעות tfhub.dev/deepmind/i3d-kinetics-400/1 מודול. מודלים נוספים כדי לזהות פעולות קטעי וידאו ניתן למצוא כאן .
המודל הבסיסי מתואר בעיתון " קוו ואדיס, הכרה פעולה? חדש דגם ובמערך הנתונים קינטיקס " על ידי ז'ואאו Carreira ואנדרו זיסרמן. המאמר פורסם ב-arXiv במאי 2017, ופורסם כמאמר כנס CVPR 2017. קוד המקור זמין לציבור על GitHub .
"Quo Vadis" הציג ארכיטקטורה חדשה לסיווג וידאו, ה-Inflated 3D Convnet או I3D. ארכיטקטורה זו השיגה תוצאות מתקדמות במערך הנתונים של UCF101 ו-HMDB51 מתוך כוונון עדין של מודלים אלה. מודלים I3D מראש מאומן על קינטיקס גם במקום הראשון CVPR 2017 האתגר חידונים .
המודול המקורי הוכשר על קינטיקה-400 dateset ויודע על 400 פעולות שונות. תוויות לפעולות אלה ניתן למצוא את קובץ מפת התווית .
ב-Colab הזה נשתמש בו לזהות פעילויות בסרטונים מתוך מערך נתונים של UCF101.
להכין
pip install -q imageiopip install -q opencv-pythonpip install -q git+https://github.com/tensorflow/docs
ייבא את המודולים הדרושים
# TensorFlow and TF-Hub modules.
from absl import logging
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed
logging.set_verbosity(logging.ERROR)
# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np
# Some modules to display an animation using imageio.
import imageio
from IPython import display
from urllib import request # requires python3
פונקציות עוזר עבור מערך הנתונים של UCF101
# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()
def list_ucf_videos():
"""Lists videos available in UCF101 dataset."""
global _VIDEO_LIST
if not _VIDEO_LIST:
index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
videos = re.findall("(v_[\w_]+\.avi)", index)
_VIDEO_LIST = sorted(set(videos))
return list(_VIDEO_LIST)
def fetch_ucf_video(video):
"""Fetchs a video and cache into local filesystem."""
cache_path = os.path.join(_CACHE_DIR, video)
if not os.path.exists(cache_path):
urlpath = request.urljoin(UCF_ROOT, video)
print("Fetching %s => %s" % (urlpath, cache_path))
data = request.urlopen(urlpath, context=unverified_context).read()
open(cache_path, "wb").write(data)
return cache_path
# Utilities to open video files using CV2
def crop_center_square(frame):
y, x = frame.shape[0:2]
min_dim = min(y, x)
start_x = (x // 2) - (min_dim // 2)
start_y = (y // 2) - (min_dim // 2)
return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]
def load_video(path, max_frames=0, resize=(224, 224)):
cap = cv2.VideoCapture(path)
frames = []
try:
while True:
ret, frame = cap.read()
if not ret:
break
frame = crop_center_square(frame)
frame = cv2.resize(frame, resize)
frame = frame[:, :, [2, 1, 0]]
frames.append(frame)
if len(frames) == max_frames:
break
finally:
cap.release()
return np.array(frames) / 255.0
def to_gif(images):
converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
imageio.mimsave('./animation.gif', converted_images, fps=25)
return embed.embed_file('./animation.gif')
קבל את התוויות Kinetics-400
# Get the kinetics-400 action labels from the GitHub repository.
KINETICS_URL = "https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt"
with request.urlopen(KINETICS_URL) as obj:
labels = [line.decode("utf-8").strip() for line in obj.readlines()]
print("Found %d labels." % len(labels))
Found 400 labels.
שימוש במערך הנתונים של UCF101
# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()
categories = {}
for video in ucf_videos:
category = video[2:-12]
if category not in categories:
categories[category] = []
categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))
for category, sequences in categories.items():
summary = ", ".join(sequences[:2])
print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))
Found 13320 videos in 101 categories. ApplyEyeMakeup 145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...) ApplyLipstick 114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...) Archery 145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...) BabyCrawling 132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...) BalanceBeam 108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...) BandMarching 155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...) BaseballPitch 150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...) BasketballDunk 131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...) Basketball 134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...) BenchPress 160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...) Biking 134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...) Billiards 150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...) BlowDryHair 131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...) BlowingCandles 109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...) BodyWeightSquats 112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...) Bowling 155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...) BoxingPunchingBag 163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...) BoxingSpeedBag 134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...) BreastStroke 101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...) BrushingTeeth 131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...) CleanAndJerk 112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...) CliffDiving 138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...) CricketBowling 139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...) CricketShot 167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...) CuttingInKitchen 110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...) Diving 150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...) Drumming 161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...) Fencing 111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...) FieldHockeyPenalty 126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...) FloorGymnastics 125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...) FrisbeeCatch 126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...) FrontCrawl 137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...) GolfSwing 139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...) Haircut 130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...) HammerThrow 150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...) Hammering 140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...) HandstandPushups 128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...) HandstandWalking 111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...) HeadMassage 147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...) HighJump 123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...) HorseRace 124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...) HorseRiding 164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...) HulaHoop 125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...) IceDancing 158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...) JavelinThrow 117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...) JugglingBalls 121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...) JumpRope 144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...) JumpingJack 123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...) Kayaking 141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...) Knitting 123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...) LongJump 131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...) Lunges 127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...) MilitaryParade 125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...) Mixing 136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...) MoppingFloor 110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...) Nunchucks 132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...) ParallelBars 114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...) PizzaTossing 113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...) PlayingCello 164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...) PlayingDaf 151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...) PlayingDhol 164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...) PlayingFlute 155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...) PlayingGuitar 160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...) PlayingPiano 105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...) PlayingSitar 157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...) PlayingTabla 111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...) PlayingViolin 100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...) PoleVault 149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...) PommelHorse 123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...) PullUps 100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...) Punch 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...) PushUps 102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...) Rafting 111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...) RockClimbingIndoor 144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...) RopeClimbing 119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...) Rowing 137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...) SalsaSpin 133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...) ShavingBeard 161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...) Shotput 144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...) SkateBoarding 120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...) Skiing 135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...) Skijet 100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...) SkyDiving 110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...) SoccerJuggling 147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...) SoccerPenalty 137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...) StillRings 112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...) SumoWrestling 116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...) Surfing 126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...) Swing 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...) TableTennisShot 140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...) TaiChi 100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...) TennisSwing 166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...) ThrowDiscus 130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...) TrampolineJumping 119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...) Typing 136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...) UnevenBars 104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...) VolleyballSpiking 116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...) WalkingWithDog 123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...) WallPushups 130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...) WritingOnBoard 152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...) YoYo 128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)
# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)
Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmp/tmp4wns8m2b/v_CricketShot_g04_c02.avi
sample_video.shape
(116, 224, 224, 3)
i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']
הפעל את מודל ה-id3 והדפיס את 5 תחזיות הפעולה המובילות.
def predict(sample_video):
# Add a batch axis to the sample video.
model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]
logits = i3d(model_input)['default'][0]
probabilities = tf.nn.softmax(logits)
print("Top 5 actions:")
for i in np.argsort(probabilities)[::-1][:5]:
print(f" {labels[i]:22}: {probabilities[i] * 100:5.2f}%")
predict(sample_video)
Top 5 actions: playing cricket : 97.77% skateboarding : 0.71% robot dancing : 0.56% roller skating : 0.56% golf putting : 0.13%
עכשיו מנסים וידאו חדש, מתוך: https://commons.wikimedia.org/wiki/Category : Videos_of_sports
מה דעתך על זה וידאו על ידי פטריק ג'ילט:
curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 55.0M 100 55.0M 0 0 21.9M 0 0:00:02 0:00:02 --:--:-- 21.9M
video_path = "End_of_a_jam.ogv"
sample_video = load_video(video_path)[:100]
sample_video.shape
(100, 224, 224, 3)
to_gif(sample_video)
predict(sample_video)
Top 5 actions: roller skating : 96.85% playing volleyball : 1.63% skateboarding : 0.21% playing ice hockey : 0.20% playing basketball : 0.16%