Light Echo and Star Detection
In this tutorial, we will use the VGG16 AI model to detect light echoes and stars in images. We will train the model on a sample dataset and explore how the model makes predictions by analyzing the feature maps.
Exercises
Load the Dataset
Train the Model
Make predictions and evaulation
Explore the feature maps
tSNE scatter plot in feature space
from google.colab import drive
drive.mount('/drive', force_remount=True)
Mounted at /drive
cd '/drive/MyDrive/oxford_workshop/'/drive/MyDrive/oxford_workshop
ls dataset/ python_intro.ipynb
ec.mp4 README.txt
faster_rcnn.ipynb torch_intro.ipynb
LEsim.ipynb vgg16.ipynb
'Oxford workshop Apr 24 2026-RC-Apr8.pptx'
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
import torchvision
from torchvision import models, transforms, datasets
from torchsummary import summary
from sklearn.manifold import TSNE
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")dataset¶
image_trans = transforms.Compose([transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
image_dataset = datasets.ImageFolder(root='./dataset/vgg/train1/', transform=image_trans)
dataloader = DataLoader(image_dataset, batch_size=4, shuffle=True, num_workers=2)
image_dataset_test = datasets.ImageFolder(root='./dataset/vgg/test1/', transform=image_trans)
dataloader_test = DataLoader(image_dataset_test, batch_size=4, shuffle=True, num_workers=2)
image_dataset.classes
#image_dataset.__dict__['LE', 'star']images, labels = next( iter(dataloader) )images.shape, labels.shape(torch.Size([4, 3, 224, 224]), torch.Size([4]))labelstensor([1, 0, 1, 0])def plot_batch(images, labels):
fig, axs = plt.subplots(1, 4, figsize=(8, 5), sharey=True)
axs = axs.flatten()
for i in range(4):
axs[i].imshow(images[i, 0, :, :], origin='lower', cmap='gray', interpolation=None);
axs[i].set_title(f'label={labels[i]}')
return fig
plot_batch(images, labels);
train the model¶
Exercise: load the pre-trained vgg16 model, and train on the dataset
# Load the pre-trained VGG16 model
model = models.vgg16(weights=models.VGG16_Weights.IMAGENET1K_V1)
Downloading: "https://download.pytorch.org/models/vgg16-397923af.pth" to /root/.cache/torch/hub/checkpoints/vgg16-397923af.pth
100%|██████████| 528M/528M [00:05<00:00, 108MB/s]
modelVGG(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): ReLU(inplace=True)
(2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): ReLU(inplace=True)
(4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(6): ReLU(inplace=True)
(7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(8): ReLU(inplace=True)
(9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(11): ReLU(inplace=True)
(12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(13): ReLU(inplace=True)
(14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(15): ReLU(inplace=True)
(16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(18): ReLU(inplace=True)
(19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(20): ReLU(inplace=True)
(21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(22): ReLU(inplace=True)
(23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(25): ReLU(inplace=True)
(26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(27): ReLU(inplace=True)
(28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(29): ReLU(inplace=True)
(30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
)
(avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
(classifier): Sequential(
(0): Linear(in_features=25088, out_features=4096, bias=True)
(1): ReLU(inplace=True)
(2): Dropout(p=0.5, inplace=False)
(3): Linear(in_features=4096, out_features=4096, bias=True)
(4): ReLU(inplace=True)
(5): Dropout(p=0.5, inplace=False)
(6): Linear(in_features=4096, out_features=1000, bias=True)
)
)# modify the last layer in classifier to match number of classes, 2 for our case
model.classifier[6] = nn.Linear(model.classifier[6].in_features, 2)model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.00001)# train the model
num_epochs = 20
loss_run = []
model.train()
for epoch in range(num_epochs):
for inputs, labels in dataloader:
# input shape [4 batchsize, 3, 224, 224], labels shape [4]
inputs = inputs.to(device)
labels = labels.to(device)
# forward
outputs = model(inputs)
loss = criterion(outputs, labels)
# backward
loss.backward()
optimizer.step()
optimizer.zero_grad()
loss_run.append(loss.item())
print(f'epoch {epoch}, loss {loss.item()}')
epoch 0, loss 0.6254315376281738
epoch 0, loss 0.6020292043685913
epoch 1, loss 0.4947372078895569
epoch 1, loss 0.7946991920471191
epoch 2, loss 0.37016546726226807
epoch 2, loss 0.39522048830986023
epoch 3, loss 0.2775701880455017
epoch 3, loss 0.46683305501937866
epoch 4, loss 0.34009212255477905
epoch 4, loss 0.3171534240245819
epoch 5, loss 0.28441351652145386
epoch 5, loss 0.2811765968799591
epoch 6, loss 0.36127975583076477
epoch 6, loss 0.09791828691959381
epoch 7, loss 0.1904105395078659
epoch 7, loss 0.10091759264469147
epoch 8, loss 0.10752193629741669
epoch 8, loss 0.11787524819374084
epoch 9, loss 0.0940801352262497
epoch 9, loss 0.04970155656337738
epoch 10, loss 0.04652848094701767
epoch 10, loss 0.036169927567243576
epoch 11, loss 0.0615931935608387
epoch 11, loss 0.0417410172522068
epoch 12, loss 0.037446606904268265
epoch 12, loss 0.029542671516537666
epoch 13, loss 0.016140718013048172
epoch 13, loss 0.00974899623543024
epoch 14, loss 0.024135839194059372
epoch 14, loss 0.003801334649324417
epoch 15, loss 0.009364629164338112
epoch 15, loss 0.004716761410236359
epoch 16, loss 0.004407037515193224
epoch 16, loss 0.005950522609055042
epoch 17, loss 0.0013311951188370585
epoch 17, loss 0.0041243210434913635
epoch 18, loss 0.0018342951079830527
epoch 18, loss 0.0028033240232616663
epoch 19, loss 0.000781133770942688
epoch 19, loss 0.0014592738589271903
plt.plot(loss_run)
plt.xlabel('epoch')
plt.ylabel('loss')
# if not able to run on GPU,
# uncomment this cell to load the trained weights
#
# model.load_state_dict(torch.load("vgg_trained.pth", map_location=device))
# model.to(device)
# model.eval() # always set to eval mode before inference
make prediction¶
images, labels = next(iter(dataloader_test))
fig, axs = plt.subplots(1, 4, figsize=(8, 5), sharey=True)
axs = axs.flatten()
for i in range(4):
axs[i].imshow(images[i, 0, :, :], origin='lower', cmap='gray', interpolation=None);
axs[i].set_title(f'label={labels[i]}')
model.eval();images = images.to(device)outputs = model(images).softmax(dim=1)
outputstensor([[0.9986, 0.0014],
[0.0127, 0.9873],
[0.7947, 0.2053],
[0.9639, 0.0361]], device='cuda:0', grad_fn=<SoftmaxBackward0>)_, preds = torch.max(outputs, 1)
predstensor([0, 1, 0, 0], device='cuda:0')labelstensor([1, 0, 1, 0])calculate the accuracy on training and test set
accuracy = (number of correct prediction) / (total number of images)
Hint: the code below is set for training set at the moment
# example for trainset
model.eval()
correct = 0
total = 0
with torch.no_grad():
for inputs, labels in dataloader:
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
total += labels.size(0)
correct += (preds == labels).sum().item()
accuracy = correct / total
print(f"Accuracy: {accuracy * 100:.2f}%")
Accuracy: 100.00%
Explore features¶
The method of visualizing feature maps by optimizing the input image involves generating an image that maximizes the activation of a specific layer or neuron within a neural network. This technique helps us understand what patterns or features a particular part of the network is sensitive to.
Steps
Select a target layer
Use the forward hook to capture activations
Initial a random image that require grads
Adjust the input image in the direction of the gradients using an optimizer
Visualize the optimized image
summary(model, (3, 244, 244))----------------------------------------------------------------
Layer (type) Output Shape Param #
================================================================
Conv2d-1 [-1, 64, 244, 244] 1,792
ReLU-2 [-1, 64, 244, 244] 0
Conv2d-3 [-1, 64, 244, 244] 36,928
ReLU-4 [-1, 64, 244, 244] 0
MaxPool2d-5 [-1, 64, 122, 122] 0
Conv2d-6 [-1, 128, 122, 122] 73,856
ReLU-7 [-1, 128, 122, 122] 0
Conv2d-8 [-1, 128, 122, 122] 147,584
ReLU-9 [-1, 128, 122, 122] 0
MaxPool2d-10 [-1, 128, 61, 61] 0
Conv2d-11 [-1, 256, 61, 61] 295,168
ReLU-12 [-1, 256, 61, 61] 0
Conv2d-13 [-1, 256, 61, 61] 590,080
ReLU-14 [-1, 256, 61, 61] 0
Conv2d-15 [-1, 256, 61, 61] 590,080
ReLU-16 [-1, 256, 61, 61] 0
MaxPool2d-17 [-1, 256, 30, 30] 0
Conv2d-18 [-1, 512, 30, 30] 1,180,160
ReLU-19 [-1, 512, 30, 30] 0
Conv2d-20 [-1, 512, 30, 30] 2,359,808
ReLU-21 [-1, 512, 30, 30] 0
Conv2d-22 [-1, 512, 30, 30] 2,359,808
ReLU-23 [-1, 512, 30, 30] 0
MaxPool2d-24 [-1, 512, 15, 15] 0
Conv2d-25 [-1, 512, 15, 15] 2,359,808
ReLU-26 [-1, 512, 15, 15] 0
Conv2d-27 [-1, 512, 15, 15] 2,359,808
ReLU-28 [-1, 512, 15, 15] 0
Conv2d-29 [-1, 512, 15, 15] 2,359,808
ReLU-30 [-1, 512, 15, 15] 0
MaxPool2d-31 [-1, 512, 7, 7] 0
AdaptiveAvgPool2d-32 [-1, 512, 7, 7] 0
Linear-33 [-1, 4096] 102,764,544
ReLU-34 [-1, 4096] 0
Dropout-35 [-1, 4096] 0
Linear-36 [-1, 4096] 16,781,312
ReLU-37 [-1, 4096] 0
Dropout-38 [-1, 4096] 0
Linear-39 [-1, 2] 8,194
================================================================
Total params: 134,268,738
Trainable params: 134,268,738
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.68
Forward/backward pass size (MB): 258.50
Params size (MB): 512.19
Estimated Total Size (MB): 771.38
----------------------------------------------------------------
def get_layer_output(model, input_image, layer_index=29,):
"""get the output from a layer
input_image: shape [batch, 3, 244, 244]"""
input_image = input_image.to(device)
# Dictionary to store the output from the specified layer
outputs = {}
def hook_fn(module, input, output):
"""register to layer, run during forward"""
outputs["layer_output"] = output
# Register the hook
hook = model.features[layer_index].register_forward_hook(hook_fn)
_ = model(input_image) # Forward pass to get the activations
# Remove the hook after optimization
hook.remove()
layer_output = outputs["layer_output"].detach().cpu().numpy()
return layer_outputdef get_featuremap(model=model,
input_data=None,
layer_index=29, feature_map_index=0, num_iterations = 100, lr=0.01):
"""optimize an random input for a specific layer to get the high activations,
"""
# Step 1: Create a random image
# Initialize a random image tensor with required shape and allow gradient computation
input_image = torch.randn((1, 3, 224, 224), requires_grad=True, device=device)
if input_data!=None:
print('initial with input data')
input_image.data = input_data.to(device).unsqueeze(0)
# Step 2: Load the pre-trained VGG16 model in evaluation mode
# Step 3: Register a forward hook to access the features of a specific layer (e.g., layer 10)
#layer_index = 29
#feature_map_index = 12 # Specify the feature map index you want to maximize
# Dictionary to store the output from the specified layer
outputs = {}
def hook_fn(module, input, output):
"""register to layer, run during forward"""
outputs["layer_output"] = output
# Register the hook
hook = model.features[layer_index].register_forward_hook(hook_fn)
# Step 4: Define the optimizer and the loss function
# Use an optimizer to change the pixel values of the input image
optimizer = torch.optim.Adam([input_image], lr=lr)
# Number of iterations for optimization
#num_iterations = 100
# Optimization loop
for i in range(num_iterations):
optimizer.zero_grad() # Clear previous gradients
_ = model(input_image) # Forward pass to get the activations
# Get the activation of the specific feature map
layer_output = outputs["layer_output"]
feature_map_activation = layer_output[0, feature_map_index] # Accessing the desired feature map
# Calculate the mean activation of the feature map
loss = - torch.mean(feature_map_activation) # Negate to maximize
# Perform backpropagation and optimize the image
loss.backward()
optimizer.step()
#print('grad', type(input_image.grad))
# Clip the values of the image tensor to keep them in the valid range
with torch.no_grad():
input_image.clamp_(0, 1)
if i % 10 == 0:
print(f"Iteration {i}, Loss: {loss.item()}")
# Remove the hook after optimization
hook.remove()
# Step 5: Display the optimized image
# Convert the optimized image tensor to a format suitable for displaying
optimized_image = input_image.detach().cpu().squeeze() # Remove batch dimension and move to CPU
optimized_image = optimized_image.permute(1, 2, 0) # Convert from [C, H, W] to [H, W, C]
optimized_image = optimized_image.numpy()
return optimized_image, input_image
img, labels = next(iter(dataloader))
plot_batch(img, labels);
# Select a layer
layer_idx = 23
model.features[layer_idx]
layer_output = get_layer_output(model, img, layer_index=layer_idx)
layer_output.shape
# show one of layer
#plt.imshow(layer_output[0, 20, :, :],)(4, 512, 14, 14)layer_output_mean = layer_output.mean(axis=3).mean(axis=2)layer_output_mean.shape(4, 512)# activation values in feature space
plt.plot(layer_output_mean[0], label=f'label={labels.numpy()[0]}')
#plt.plot(layer_output_mean[1], alpha=0.5,)
#plt.plot(layer_output_mean[2], alpha=0.5)
plt.plot(layer_output_mean[3], alpha=0.5, label=f'label={labels.numpy()[3]}')
plt.xlabel('features')
plt.ylabel('activation')
plt.legend()
np.argsort(layer_output_mean[1])[::-1][:5]array([341, 235, 75, 212, 110])np.argsort(layer_output_mean[0])[::-1][:5]
#layer_output_mean[0] [ np.argsort(layer_output_mean[0])[::-1][:5] ]
array([341, 235, 58, 213, 212])fmaps = {}
#feature_list = [119, 65, 246, 385, 172]
#feature_list = [36, 45, 8, 18, 5]
feature_list = np.argsort(layer_output_mean[0])[::-1][:5]
#layer_idx = 30
for i in feature_list:
#idx = 303
optimized_image, input_image = get_featuremap(model=model,
#input_data=img[0].data,
layer_index=layer_idx, feature_map_index=i,
num_iterations=300, lr=0.05)
fmaps[i] = optimized_imageIteration 0, Loss: -1.9062684774398804
Iteration 10, Loss: -24.231983184814453
Iteration 20, Loss: -47.09925842285156
Iteration 30, Loss: -68.3695068359375
Iteration 40, Loss: -85.72322082519531
Iteration 50, Loss: -99.1561050415039
Iteration 60, Loss: -110.15039825439453
Iteration 70, Loss: -119.00281524658203
Iteration 80, Loss: -125.68888092041016
Iteration 90, Loss: -131.0581512451172
Iteration 100, Loss: -135.3539276123047
Iteration 110, Loss: -138.994140625
Iteration 120, Loss: -142.04710388183594
Iteration 130, Loss: -144.69400024414062
Iteration 140, Loss: -146.9715118408203
Iteration 150, Loss: -148.92926025390625
Iteration 160, Loss: -150.63133239746094
Iteration 170, Loss: -152.11029052734375
Iteration 180, Loss: -153.43954467773438
Iteration 190, Loss: -154.69276428222656
Iteration 200, Loss: -155.82261657714844
Iteration 210, Loss: -156.82640075683594
Iteration 220, Loss: -157.70089721679688
Iteration 230, Loss: -158.5077362060547
Iteration 240, Loss: -159.15660095214844
Iteration 250, Loss: -159.7874298095703
Iteration 260, Loss: -160.36268615722656
Iteration 270, Loss: -160.91259765625
Iteration 280, Loss: -161.39401245117188
Iteration 290, Loss: -161.860107421875
Iteration 0, Loss: -0.5898014903068542
Iteration 10, Loss: -34.004302978515625
Iteration 20, Loss: -68.76383972167969
Iteration 30, Loss: -93.08207702636719
Iteration 40, Loss: -107.69642639160156
Iteration 50, Loss: -116.85852813720703
Iteration 60, Loss: -122.87872314453125
Iteration 70, Loss: -126.97494506835938
Iteration 80, Loss: -129.9715576171875
Iteration 90, Loss: -132.2899932861328
Iteration 100, Loss: -134.08502197265625
Iteration 110, Loss: -135.4917449951172
Iteration 120, Loss: -136.60888671875
Iteration 130, Loss: -137.5557403564453
Iteration 140, Loss: -138.33126831054688
Iteration 150, Loss: -139.0004119873047
Iteration 160, Loss: -139.58169555664062
Iteration 170, Loss: -140.1088104248047
Iteration 180, Loss: -140.57577514648438
Iteration 190, Loss: -141.00885009765625
Iteration 200, Loss: -141.40338134765625
Iteration 210, Loss: -141.75807189941406
Iteration 220, Loss: -142.07818603515625
Iteration 230, Loss: -142.3873291015625
Iteration 240, Loss: -142.6656951904297
Iteration 250, Loss: -142.91061401367188
Iteration 260, Loss: -143.14337158203125
Iteration 270, Loss: -143.37461853027344
Iteration 280, Loss: -143.60092163085938
Iteration 290, Loss: -143.81956481933594
Iteration 0, Loss: -0.3987900912761688
Iteration 10, Loss: -33.58445358276367
Iteration 20, Loss: -75.1875991821289
Iteration 30, Loss: -107.14047241210938
Iteration 40, Loss: -127.81842803955078
Iteration 50, Loss: -141.76229858398438
Iteration 60, Loss: -151.52359008789062
Iteration 70, Loss: -158.3658447265625
Iteration 80, Loss: -163.71759033203125
Iteration 90, Loss: -168.22715759277344
Iteration 100, Loss: -171.81126403808594
Iteration 110, Loss: -174.85379028320312
Iteration 120, Loss: -177.25509643554688
Iteration 130, Loss: -179.2048797607422
Iteration 140, Loss: -180.8373260498047
Iteration 150, Loss: -182.19610595703125
Iteration 160, Loss: -183.31187438964844
Iteration 170, Loss: -184.349609375
Iteration 180, Loss: -185.20970153808594
Iteration 190, Loss: -185.9492950439453
Iteration 200, Loss: -186.64625549316406
Iteration 210, Loss: -187.2552947998047
Iteration 220, Loss: -187.82598876953125
Iteration 230, Loss: -188.3565673828125
Iteration 240, Loss: -188.86582946777344
Iteration 250, Loss: -189.39324951171875
Iteration 260, Loss: -189.82476806640625
Iteration 270, Loss: -190.2435760498047
Iteration 280, Loss: -190.6144256591797
Iteration 290, Loss: -190.96975708007812
Iteration 0, Loss: -1.2917197942733765
Iteration 10, Loss: -24.031652450561523
Iteration 20, Loss: -44.082794189453125
Iteration 30, Loss: -59.33910369873047
Iteration 40, Loss: -70.15280151367188
Iteration 50, Loss: -78.0279541015625
Iteration 60, Loss: -83.91910552978516
Iteration 70, Loss: -88.3453598022461
Iteration 80, Loss: -91.8717269897461
Iteration 90, Loss: -94.75020599365234
Iteration 100, Loss: -97.06110382080078
Iteration 110, Loss: -98.98431396484375
Iteration 120, Loss: -100.62852478027344
Iteration 130, Loss: -102.09689331054688
Iteration 140, Loss: -103.40791320800781
Iteration 150, Loss: -104.50699615478516
Iteration 160, Loss: -105.45549774169922
Iteration 170, Loss: -106.28706359863281
Iteration 180, Loss: -107.08611297607422
Iteration 190, Loss: -107.81471252441406
Iteration 200, Loss: -108.52081298828125
Iteration 210, Loss: -109.23123168945312
Iteration 220, Loss: -109.89757537841797
Iteration 230, Loss: -110.51384735107422
Iteration 240, Loss: -111.09534454345703
Iteration 250, Loss: -111.6358413696289
Iteration 260, Loss: -112.1552505493164
Iteration 270, Loss: -112.67794799804688
Iteration 280, Loss: -113.18929290771484
Iteration 290, Loss: -113.68416595458984
Iteration 0, Loss: -0.7070236206054688
Iteration 10, Loss: -58.975624084472656
Iteration 20, Loss: -113.57168579101562
Iteration 30, Loss: -146.1985321044922
Iteration 40, Loss: -164.8509063720703
Iteration 50, Loss: -176.31166076660156
Iteration 60, Loss: -183.99172973632812
Iteration 70, Loss: -189.2761993408203
Iteration 80, Loss: -193.11288452148438
Iteration 90, Loss: -196.04641723632812
Iteration 100, Loss: -198.3473663330078
Iteration 110, Loss: -200.1524658203125
Iteration 120, Loss: -201.70257568359375
Iteration 130, Loss: -203.0226593017578
Iteration 140, Loss: -204.13934326171875
Iteration 150, Loss: -205.14279174804688
Iteration 160, Loss: -206.0523681640625
Iteration 170, Loss: -206.84156799316406
Iteration 180, Loss: -207.53306579589844
Iteration 190, Loss: -208.1106414794922
Iteration 200, Loss: -208.66490173339844
Iteration 210, Loss: -209.18104553222656
Iteration 220, Loss: -209.66159057617188
Iteration 230, Loss: -210.0846405029297
Iteration 240, Loss: -210.46424865722656
Iteration 250, Loss: -210.80859375
Iteration 260, Loss: -211.1361541748047
Iteration 270, Loss: -211.41001892089844
Iteration 280, Loss: -211.68641662597656
Iteration 290, Loss: -211.94024658203125
n = len(feature_list)
fig, axs = plt.subplots(1, n-1, figsize=(20, 5))
for i in range(n-1):
axs[i].imshow(fmaps[ feature_list[i] ])
axs[i].set_title(feature_list[i] )
axs[i].axis('off')
Task: plot feature maps from an early and a deeper layers of VGG16, what differences do you observe?¶
Hint: layer_idx
# Select a layer
layer_idx = 4
layer_output = get_layer_output(model, img, layer_index=layer_idx)
layer_output_mean = layer_output.mean(axis=3).mean(axis=2)
fmaps = {}
feature_list = np.argsort(layer_output_mean[0])[::-1][:5]
for i in feature_list:
#idx = 303
optimized_image, input_image = get_featuremap(model=model,
#input_data=img[0].data,
layer_index=layer_idx, feature_map_index=i,
num_iterations=300, lr=0.05)
fmaps[i] = optimized_image
Iteration 0, Loss: -7.725295066833496
Iteration 10, Loss: -9.546738624572754
Iteration 20, Loss: -12.961481094360352
Iteration 30, Loss: -13.68641471862793
Iteration 40, Loss: -13.823478698730469
Iteration 50, Loss: -13.895867347717285
Iteration 60, Loss: -13.935245513916016
Iteration 70, Loss: -13.96109390258789
Iteration 80, Loss: -13.977987289428711
Iteration 90, Loss: -13.988146781921387
Iteration 100, Loss: -13.993927001953125
Iteration 110, Loss: -13.999244689941406
Iteration 120, Loss: -14.001490592956543
Iteration 130, Loss: -14.002923965454102
Iteration 140, Loss: -14.004603385925293
Iteration 150, Loss: -14.005847930908203
Iteration 160, Loss: -14.006482124328613
Iteration 170, Loss: -14.006726264953613
Iteration 180, Loss: -14.007232666015625
Iteration 190, Loss: -14.007513046264648
Iteration 200, Loss: -14.007668495178223
Iteration 210, Loss: -14.00778865814209
Iteration 220, Loss: -14.008382797241211
Iteration 230, Loss: -14.008574485778809
Iteration 240, Loss: -14.008631706237793
Iteration 250, Loss: -14.008671760559082
Iteration 260, Loss: -14.008699417114258
Iteration 270, Loss: -14.008719444274902
Iteration 280, Loss: -14.00874137878418
Iteration 290, Loss: -14.008756637573242
Iteration 0, Loss: -7.331858158111572
Iteration 10, Loss: -9.13580322265625
Iteration 20, Loss: -12.386938095092773
Iteration 30, Loss: -13.100685119628906
Iteration 40, Loss: -13.234023094177246
Iteration 50, Loss: -13.301468849182129
Iteration 60, Loss: -13.341711044311523
Iteration 70, Loss: -13.367366790771484
Iteration 80, Loss: -13.385906219482422
Iteration 90, Loss: -13.398233413696289
Iteration 100, Loss: -13.40815544128418
Iteration 110, Loss: -13.413633346557617
Iteration 120, Loss: -13.417095184326172
Iteration 130, Loss: -13.420884132385254
Iteration 140, Loss: -13.423714637756348
Iteration 150, Loss: -13.425424575805664
Iteration 160, Loss: -13.42732048034668
Iteration 170, Loss: -13.428436279296875
Iteration 180, Loss: -13.42899227142334
Iteration 190, Loss: -13.429566383361816
Iteration 200, Loss: -13.430143356323242
Iteration 210, Loss: -13.43038558959961
Iteration 220, Loss: -13.430517196655273
Iteration 230, Loss: -13.430596351623535
Iteration 240, Loss: -13.430639266967773
Iteration 250, Loss: -13.430667877197266
Iteration 260, Loss: -13.430688858032227
Iteration 270, Loss: -13.430710792541504
Iteration 280, Loss: -13.430728912353516
Iteration 290, Loss: -13.430741310119629
Iteration 0, Loss: -4.947256088256836
Iteration 10, Loss: -6.150274753570557
Iteration 20, Loss: -8.487728118896484
Iteration 30, Loss: -8.98651123046875
Iteration 40, Loss: -9.071355819702148
Iteration 50, Loss: -9.112550735473633
Iteration 60, Loss: -9.137166023254395
Iteration 70, Loss: -9.15345573425293
Iteration 80, Loss: -9.162957191467285
Iteration 90, Loss: -9.169816970825195
Iteration 100, Loss: -9.174238204956055
Iteration 110, Loss: -9.17713737487793
Iteration 120, Loss: -9.180014610290527
Iteration 130, Loss: -9.182173728942871
Iteration 140, Loss: -9.18353271484375
Iteration 150, Loss: -9.185184478759766
Iteration 160, Loss: -9.186067581176758
Iteration 170, Loss: -9.18659496307373
Iteration 180, Loss: -9.187129974365234
Iteration 190, Loss: -9.187437057495117
Iteration 200, Loss: -9.1875581741333
Iteration 210, Loss: -9.187623023986816
Iteration 220, Loss: -9.187801361083984
Iteration 230, Loss: -9.188044548034668
Iteration 240, Loss: -9.188264846801758
Iteration 250, Loss: -9.188314437866211
Iteration 260, Loss: -9.18834400177002
Iteration 270, Loss: -9.188453674316406
Iteration 280, Loss: -9.188498497009277
Iteration 290, Loss: -9.188514709472656
Iteration 0, Loss: -6.078230857849121
Iteration 10, Loss: -7.807146072387695
Iteration 20, Loss: -10.847952842712402
Iteration 30, Loss: -11.606754302978516
Iteration 40, Loss: -11.82083511352539
Iteration 50, Loss: -11.94446849822998
Iteration 60, Loss: -12.022384643554688
Iteration 70, Loss: -12.073019981384277
Iteration 80, Loss: -12.110628128051758
Iteration 90, Loss: -12.141491889953613
Iteration 100, Loss: -12.161348342895508
Iteration 110, Loss: -12.172054290771484
Iteration 120, Loss: -12.178608894348145
Iteration 130, Loss: -12.184152603149414
Iteration 140, Loss: -12.18718433380127
Iteration 150, Loss: -12.190113067626953
Iteration 160, Loss: -12.192497253417969
Iteration 170, Loss: -12.195284843444824
Iteration 180, Loss: -12.19687271118164
Iteration 190, Loss: -12.197218894958496
Iteration 200, Loss: -12.197916030883789
Iteration 210, Loss: -12.198067665100098
Iteration 220, Loss: -12.198122024536133
Iteration 230, Loss: -12.198160171508789
Iteration 240, Loss: -12.19819450378418
Iteration 250, Loss: -12.198568344116211
Iteration 260, Loss: -12.199371337890625
Iteration 270, Loss: -12.199492454528809
Iteration 280, Loss: -12.199557304382324
Iteration 290, Loss: -12.199708938598633
Iteration 0, Loss: -6.101819038391113
Iteration 10, Loss: -8.104884147644043
Iteration 20, Loss: -11.670635223388672
Iteration 30, Loss: -12.529134750366211
Iteration 40, Loss: -12.743587493896484
Iteration 50, Loss: -12.868684768676758
Iteration 60, Loss: -12.94381046295166
Iteration 70, Loss: -12.991358757019043
Iteration 80, Loss: -13.022148132324219
Iteration 90, Loss: -13.045230865478516
Iteration 100, Loss: -13.064425468444824
Iteration 110, Loss: -13.076627731323242
Iteration 120, Loss: -13.084407806396484
Iteration 130, Loss: -13.088236808776855
Iteration 140, Loss: -13.091294288635254
Iteration 150, Loss: -13.093196868896484
Iteration 160, Loss: -13.095806121826172
Iteration 170, Loss: -13.097892761230469
Iteration 180, Loss: -13.098475456237793
Iteration 190, Loss: -13.099540710449219
Iteration 200, Loss: -13.100578308105469
Iteration 210, Loss: -13.101180076599121
Iteration 220, Loss: -13.10130786895752
Iteration 230, Loss: -13.10139274597168
Iteration 240, Loss: -13.101442337036133
Iteration 250, Loss: -13.101485252380371
Iteration 260, Loss: -13.101502418518066
Iteration 270, Loss: -13.101511001586914
Iteration 280, Loss: -13.101517677307129
Iteration 290, Loss: -13.101816177368164
n = len(feature_list)
fig, axs = plt.subplots(1, n-1, figsize=(20, 5))
for i in range(n-1):
axs[i].imshow(fmaps[ feature_list[i] ])
axs[i].set_title(feature_list[i] )
axs[i].axis('off')
TSNE¶
t-SNE is a dimensionality reduction technique used for visualizing high-dimensional data in a lower-dimensional space, typically 2D or 3D. It is especially useful for exploring the high-dimensional datasets and understanding how data points relate to each other.
Colors = {0: 'red', 1: 'blue'}
#image_dataset_all = datasets.ImageFolder(root='./dataset/vgg/test1/', transform=image_trans)
dataloader_all = DataLoader(image_dataset_all, batch_size=4, shuffle=False, num_workers=2)
img, labels = next(iter(dataloader_all))
img.shapetorch.Size([4, 3, 224, 224])# Select a layer
layer_idx = 29
model.features[layer_idx]
layer_output = get_layer_output(model, img, layer_index=layer_idx)
layer_output_mean = layer_output.mean(axis=3).mean(axis=2)
# show one of layer
#plt.imshow(layer_output[0, 20, :, :],)
layer_output_mean.shape(4, 512)tsne = TSNE(n_components=2, random_state=0, perplexity=3)
tsne_results = tsne.fit_transform(layer_output_mean)
plt.scatter(tsne_results[:, 0], tsne_results[:, 1],
color=[Colors[i] for i in labels.numpy()])
plt.xlabel('feature 1')
plt.ylabel('feature 2')
#plt.legend()
Task: plot for all images in the folder cuts_out_all , does this reveal any clustering patterns for LE and star images?¶
Hint: the code above is set for test folder at the moment
Task: Compare scatter plots from early and later layers— which layer shows better clustering?¶
#