 TensorFlow.org पर देखें |  Google Colab में चलाएं |  गिटहब पर देखें |  नोटबुक डाउनलोड करें |  टीएफ हब मॉडल देखें |
यह Colab का उपयोग कर वीडियो डेटा में पहचानने कार्यों को दर्शाता है tfhub.dev/deepmind/i3d-kinetics-400/1 मॉड्यूल। और मॉडल वीडियो में कार्रवाई पाया जा सकता है पता लगाने के लिए यहां ।
अंतर्निहित मॉडल पत्र में वर्णित है " क्यू Vadis, कार्रवाई मान्यता? एक नई मॉडल और काइनेटिक्स डेटासेट " जोआओ Carreira और एंड्रयू ज़िसरमैन द्वारा। यह पेपर मई 2017 में arXiv पर पोस्ट किया गया था और इसे CVPR 2017 कॉन्फ़्रेंस पेपर के रूप में प्रकाशित किया गया था। स्रोत कोड पर सार्वजनिक रूप से उपलब्ध है GitHub ।
"Quo Vadis" ने वीडियो वर्गीकरण के लिए एक नया आर्किटेक्चर पेश किया, इन्फ्लेटेड 3D Convnet या I3D। इस आर्किटेक्चर ने इन मॉडलों को फाइन-ट्यूनिंग करके UCF101 और HMDB51 डेटासेट पर अत्याधुनिक परिणाम प्राप्त किए। I3D मॉडल काइनेटिक्स पर पहले से प्रशिक्षित भी CVPR 2017 में पहले रखा Charades चुनौती ।
मूल मॉड्यूल पर प्रशिक्षित किया गया था गतिकी -400 dateset और 400 विभिन्न कार्यों के बारे में जानता है। इन कार्यों के लिए लेबल में पाया जा सकता लेबल मानचित्र फ़ाइल ।
इस Colab में हम इसका उपयोग UCF101 डेटासेट से वीडियो में गतिविधियों की पहचान करने के लिए करेंगे।
सेट अप
pip install -q imageiopip install -q opencv-pythonpip install -q git+https://github.com/tensorflow/docs
आवश्यक मॉड्यूल आयात करें
# TensorFlow and TF-Hub modules.
from absl import logging
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed
logging.set_verbosity(logging.ERROR)
# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np
# Some modules to display an animation using imageio.
import imageio
from IPython import display
from urllib import request # requires python3
UCF101 डेटासेट के लिए सहायक कार्य
# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()
def list_ucf_videos():
"""Lists videos available in UCF101 dataset."""
global _VIDEO_LIST
if not _VIDEO_LIST:
index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
videos = re.findall("(v_[\w_]+\.avi)", index)
_VIDEO_LIST = sorted(set(videos))
return list(_VIDEO_LIST)
def fetch_ucf_video(video):
"""Fetchs a video and cache into local filesystem."""
cache_path = os.path.join(_CACHE_DIR, video)
if not os.path.exists(cache_path):
urlpath = request.urljoin(UCF_ROOT, video)
print("Fetching %s => %s" % (urlpath, cache_path))
data = request.urlopen(urlpath, context=unverified_context).read()
open(cache_path, "wb").write(data)
return cache_path
# Utilities to open video files using CV2
def crop_center_square(frame):
y, x = frame.shape[0:2]
min_dim = min(y, x)
start_x = (x // 2) - (min_dim // 2)
start_y = (y // 2) - (min_dim // 2)
return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]
def load_video(path, max_frames=0, resize=(224, 224)):
cap = cv2.VideoCapture(path)
frames = []
try:
while True:
ret, frame = cap.read()
if not ret:
break
frame = crop_center_square(frame)
frame = cv2.resize(frame, resize)
frame = frame[:, :, [2, 1, 0]]
frames.append(frame)
if len(frames) == max_frames:
break
finally:
cap.release()
return np.array(frames) / 255.0
def to_gif(images):
converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
imageio.mimsave('./animation.gif', converted_images, fps=25)
return embed.embed_file('./animation.gif')
कैनेटीक्स-400 लेबल प्राप्त करें
# Get the kinetics-400 action labels from the GitHub repository.
KINETICS_URL = "https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt"
with request.urlopen(KINETICS_URL) as obj:
labels = [line.decode("utf-8").strip() for line in obj.readlines()]
print("Found %d labels." % len(labels))
Found 400 labels.
UCF101 डेटासेट का उपयोग करना
# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()
categories = {}
for video in ucf_videos:
category = video[2:-12]
if category not in categories:
categories[category] = []
categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))
for category, sequences in categories.items():
summary = ", ".join(sequences[:2])
print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))
Found 13320 videos in 101 categories. ApplyEyeMakeup 145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...) ApplyLipstick 114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...) Archery 145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...) BabyCrawling 132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...) BalanceBeam 108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...) BandMarching 155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...) BaseballPitch 150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...) BasketballDunk 131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...) Basketball 134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...) BenchPress 160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...) Biking 134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...) Billiards 150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...) BlowDryHair 131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...) BlowingCandles 109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...) BodyWeightSquats 112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...) Bowling 155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...) BoxingPunchingBag 163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...) BoxingSpeedBag 134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...) BreastStroke 101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...) BrushingTeeth 131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...) CleanAndJerk 112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...) CliffDiving 138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...) CricketBowling 139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...) CricketShot 167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...) CuttingInKitchen 110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...) Diving 150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...) Drumming 161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...) Fencing 111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...) FieldHockeyPenalty 126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...) FloorGymnastics 125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...) FrisbeeCatch 126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...) FrontCrawl 137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...) GolfSwing 139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...) Haircut 130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...) HammerThrow 150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...) Hammering 140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...) HandstandPushups 128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...) HandstandWalking 111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...) HeadMassage 147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...) HighJump 123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...) HorseRace 124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...) HorseRiding 164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...) HulaHoop 125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...) IceDancing 158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...) JavelinThrow 117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...) JugglingBalls 121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...) JumpRope 144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...) JumpingJack 123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...) Kayaking 141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...) Knitting 123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...) LongJump 131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...) Lunges 127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...) MilitaryParade 125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...) Mixing 136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...) MoppingFloor 110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...) Nunchucks 132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...) ParallelBars 114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...) PizzaTossing 113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...) PlayingCello 164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...) PlayingDaf 151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...) PlayingDhol 164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...) PlayingFlute 155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...) PlayingGuitar 160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...) PlayingPiano 105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...) PlayingSitar 157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...) PlayingTabla 111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...) PlayingViolin 100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...) PoleVault 149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...) PommelHorse 123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...) PullUps 100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...) Punch 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...) PushUps 102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...) Rafting 111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...) RockClimbingIndoor 144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...) RopeClimbing 119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...) Rowing 137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...) SalsaSpin 133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...) ShavingBeard 161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...) Shotput 144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...) SkateBoarding 120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...) Skiing 135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...) Skijet 100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...) SkyDiving 110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...) SoccerJuggling 147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...) SoccerPenalty 137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...) StillRings 112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...) SumoWrestling 116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...) Surfing 126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...) Swing 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...) TableTennisShot 140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...) TaiChi 100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...) TennisSwing 166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...) ThrowDiscus 130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...) TrampolineJumping 119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...) Typing 136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...) UnevenBars 104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...) VolleyballSpiking 116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...) WalkingWithDog 123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...) WallPushups 130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...) WritingOnBoard 152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...) YoYo 128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)
# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)
Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmp/tmp4wns8m2b/v_CricketShot_g04_c02.avi
sample_video.shape
(116, 224, 224, 3)
i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']
id3 मॉडल चलाएँ और शीर्ष -5 क्रिया पूर्वानुमानों को प्रिंट करें।
def predict(sample_video):
# Add a batch axis to the sample video.
model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]
logits = i3d(model_input)['default'][0]
probabilities = tf.nn.softmax(logits)
print("Top 5 actions:")
for i in np.argsort(probabilities)[::-1][:5]:
print(f" {labels[i]:22}: {probabilities[i] * 100:5.2f}%")
predict(sample_video)
Top 5 actions: playing cricket : 97.77% skateboarding : 0.71% robot dancing : 0.56% roller skating : 0.56% golf putting : 0.13%
: अब एक नया वीडियो, से कोशिश https://commons.wikimedia.org/wiki/Category : Videos_of_sports
कैसे के बारे में इस वीडियो को पैट्रिक गिलेट द्वारा:
curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 55.0M 100 55.0M 0 0 21.9M 0 0:00:02 0:00:02 --:--:-- 21.9M
video_path = "End_of_a_jam.ogv"
sample_video = load_video(video_path)[:100]
sample_video.shape
(100, 224, 224, 3)
to_gif(sample_video)
predict(sample_video)
Top 5 actions: roller skating : 96.85% playing volleyball : 1.63% skateboarding : 0.21% playing ice hockey : 0.20% playing basketball : 0.16%