Skip to content
Snippets Groups Projects
Commit d8cbe275 authored by Sören Toschek's avatar Sören Toschek
Browse files

Breaking Changes. Buggy commit.

parent 14f08471
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
cascading_transformer_training.py
+ 5
− 3
View file @ d8cbe275
......@@ -34,6 +34,7 @@ if train_0:
5e-4,
500,
False,
True,
#load("./hierarchicalTransformer/checkpoints/high_res_lab_1_final.pt",weights_only=True),
#load("./featureVector/checkpoints/high_res_lab_1_final.pt",weights_only=True),
batch_size=4
......@@ -45,12 +46,13 @@ if train_0:
if train_1:
train_layer(
2,
"new_loss_part_2",
"new_loss_part_4",
5e-4,
100,
True,
load("./hierarchicalTransformer/checkpoints/new_loss_final.pt",weights_only=True),
load("./featureVector/checkpoints/new_loss_final.pt",weights_only=True),
False,
load("./hierarchicalTransformer/checkpoints/new_loss_part_3_final.pt",weights_only=True),
load("./featureVector/checkpoints/new_loss_part_3_final.pt",weights_only=True),
)
......
check.ipynb
+ 103
− 21
View file @ d8cbe275
%% Cell type:markdown id: tags:
A general script to visually various aspects of the voxel generation pipeline such as:
1. Correct mapping &inverse mapping between binary voxel grids and token/codeword grids.
2. Plausible upscaling of shapes.
3. Classifier of voxel shapes.
%% Cell type:code id: tags:
``` python
from codewords import *
from dataLoading import dataset
import k3d
import numpy
from dotenv import load_dotenv
from codewords import *
from dataLoading import dataset
import k3d
import numpy
from upsampling_transformer import *
from constants import dim_embedding
from feature_vector import MasterFeatureVector
from first_step_transformer import firstStepNetwork
```
%% Cell type:code id: tags:
``` python
checkpoint_path = "./hierarchicalTransformer/checkpoints/first_step_test_499.pt"
model_0 = firstStepNetwork(8)
model_0.load_state_dict(torch.load(checkpoint_path,weights_only=True))
model_0.cuda()
grids = model.sample(4)
for i in range(4):
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([8,8,8]),opacity=0.01)
plot += k3d.voxels(grids[i].cpu())
plot.display()
#Debug first step of the chain.
```
%% Output
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[2], line 7
4 model_0.load_state_dict(torch.load(checkpoint_path,weights_only=True))
FileNotFoundError Traceback (most recent call last)
Cell In[2], line 4
1 checkpoint_path = "./hierarchicalTransformer/checkpoints/first_step_test_499.pt"
3 model_0 = firstStepNetwork(8)
----> 4 model_0.load_state_dict(torch.load(checkpoint_path,weights_only=True))
6 model_0.cuda()
----> 7 grids = model.sample(4)
9 for i in range(4):
10 plot = k3d.plot()
NameError: name 'model' is not defined
7 grids = model.sample(4)
File d:\VoxelTransformer\.venv\Lib\site-packages\torch\serialization.py:791, in load(f, map_location, pickle_module, weights_only, **pickle_load_args)
788 if 'encoding' not in pickle_load_args.keys():
789 pickle_load_args['encoding'] = 'utf-8'
--> 791 with _open_file_like(f, 'rb') as opened_file:
792 if _is_zipfile(opened_file):
793 # The zipfile reader is going to advance the current file position.
794 # If we want to actually tail call to torch.jit.load, we need to
795 # reset back to the original position.
796 orig_position = opened_file.tell()
File d:\VoxelTransformer\.venv\Lib\site-packages\torch\serialization.py:271, in _open_file_like(name_or_buffer, mode)
269 def _open_file_like(name_or_buffer, mode):
270 if _is_path(name_or_buffer):
--> 271 return _open_file(name_or_buffer, mode)
272 else:
273 if 'w' in mode:
File d:\VoxelTransformer\.venv\Lib\site-packages\torch\serialization.py:252, in _open_file.__init__(self, name, mode)
251 def __init__(self, name, mode):
--> 252 super().__init__(open(name, mode))
FileNotFoundError: [Errno 2] No such file or directory: './hierarchicalTransformer/checkpoints/first_step_test_499.pt'
%% Cell type:markdown id: tags:
Fazit: sieht gut aus.
%% Cell type:code id: tags:
``` python
#debug voxel upscaling:
checkpoint_path = "./hierarchicalTransformer/checkpoints/new_loss_part_2_final.pt"
feature_vector_ckpt_path = "./featureVector/checkpoints/new_loss_part_2_final.pt"
checkpoint_path = "./hierarchicalTransformer/checkpoints/new_loss_part_4_final.pt"
feature_vector_ckpt_path = "./featureVector/checkpoints/new_loss_part_4_final.pt"
grid_size = 8
featureVector = MasterFeatureVector(1)
featureVector.load_state_dict(torch.load(feature_vector_ckpt_path,weights_only=True))
#Reminder: upsamplingVoxelTransformer resolution is somewhat independent of "grid_size", due to the chunking.
upsampling_network = UpsamplingVoxelTransformer(8,use_bce_loss=True)
upsampling_network.load_state_dict(torch.load(checkpoint_path,weights_only=True))
model = UpsampleShape(
upsampling_network,
featureVector,
1
)
n_samples = 4
print(len(dataset))
random_indices = torch.randint(0,50,[n_samples])
batch = dataset[list(random_indices)]
model.cuda()
low_res = DownScaling(2)
low_res_grids = low_res.forward(batch,flatten=False)
result = model.sample(low_res_grids.float().cuda())
numpy_voxels = result[0].detach()
grid_size_low = 16
high_res = 2*grid_size_low
upsampler = nn.Upsample(scale_factor = 2)
for i in range(n_samples):
stackable_low_res = upsampler.forward(low_res_grids[i,:,:,:].cpu().float().unsqueeze(0).unsqueeze(0)).squeeze(0).squeeze(0)
print(low_res_grids[i,:,:,:].shape)
print(result[i,:,:,:].shape)
print(stackable_low_res.shape)
combined = torch.cat((stackable_low_res,result[i,:,:,:].cpu(),batch[i].cpu()),dim=2)
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([grid_size_low,grid_size_low,grid_size_low]),opacity=0.01)
plot += k3d.voxels(combined.cpu())
plot.display()
"""
#low-res plot.
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([grid_size_low,grid_size_low,grid_size_low]),opacity=0.01)
plot += k3d.voxels(low_res_grids[i,:,:,:].cpu())
plot.display()
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([high_res,high_res,high_res]),opacity=0.01)
plot += k3d.voxels(result[i,:,:,:].cpu())
#plot += grid_filler
plot.display()
#ground truth:
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([high_res,high_res,high_res]),opacity=0.01)
plot += k3d.voxels(batch[i].cpu())
plot.display()
"""
```
%% Output
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[4], line 10
7 grid_size = 8
9 featureVector = MasterFeatureVector(1)
---> 10 featureVector.load_state_dict(torch.load(feature_vector_ckpt_path,weights_only=True))
12 #Reminder: upsamplingVoxelTransformer resolution is somewhat independent of "grid_size", due to the chunking.
13 upsampling_network = UpsamplingVoxelTransformer(8,use_bce_loss=True)
File ~/Documents/VoxelTransformer/.venv/lib/python3.11/site-packages/torch/serialization.py:1319, in load(f, map_location, pickle_module, weights_only, mmap, **pickle_load_args)
1316 if "encoding" not in pickle_load_args.keys():
1317 pickle_load_args["encoding"] = "utf-8"
-> 1319 with _open_file_like(f, "rb") as opened_file:
1320 if _is_zipfile(opened_file):
1321 # The zipfile reader is going to advance the current file position.
1322 # If we want to actually tail call to torch.jit.load, we need to
1323 # reset back to the original position.
1324 orig_position = opened_file.tell()
File ~/Documents/VoxelTransformer/.venv/lib/python3.11/site-packages/torch/serialization.py:659, in _open_file_like(name_or_buffer, mode)
657 def _open_file_like(name_or_buffer, mode):
658 if _is_path(name_or_buffer):
--> 659 return _open_file(name_or_buffer, mode)
660 else:
661 if "w" in mode:
File ~/Documents/VoxelTransformer/.venv/lib/python3.11/site-packages/torch/serialization.py:640, in _open_file.__init__(self, name, mode)
639 def __init__(self, name, mode):
--> 640 super().__init__(open(name, mode))
FileNotFoundError: [Errno 2] No such file or directory: './featureVector/checkpoints/new_loss_2_final.pt'
2000
torch.Size([16, 16, 16])
torch.Size([32, 32, 32])
torch.Size([32, 32, 32])
torch.Size([16, 16, 16])
torch.Size([32, 32, 32])
torch.Size([32, 32, 32])
torch.Size([16, 16, 16])
torch.Size([32, 32, 32])
torch.Size([32, 32, 32])
torch.Size([16, 16, 16])
torch.Size([32, 32, 32])
torch.Size([32, 32, 32])
%% Cell type:code id: tags:
``` python
```
%% Cell type:markdown id: tags:
Here we compose multiple upsampling steps and check the result.
%% Cell type:code id: tags:
``` python
checkpoint_path_0 = "./hierarchicalTransformer/checkpoints/first_step_test_499.pt"
checkpoint_path_1 = "./hierarchicalTransformer/checkpoints/lab_5_final.pt"
feature_vector_ckpt_path_1 = "./featureVector/checkpoints/lab_5_final.pt"
checkpoint_path_2 = "./hierarchicalTransformer/checkpoints/long_training_test_final.pt"
feature_vector_ckpt_path_2 = "./featureVector/checkpoints/long_training_test_final.pt"
grid_size_start = 8
#we upscale from 8³ to 16³, and then from 16³ to 32³
model_0 = firstStepNetwork(8)
model_0.load_state_dict(torch.load(checkpoint_path_0,weights_only=True))
model_0.cuda()
featureVector_1 = MasterFeatureVector(1)
featureVector_1.load_state_dict(torch.load(feature_vector_ckpt_path_1,weights_only=True))
#Reminder: upsamplingVoxelTransformer resolution is somewhat independent of "grid_size", due to the chunking.
upsampling_network_1 = UpsamplingVoxelTransformer(8)
upsampling_network_1.load_state_dict(torch.load(checkpoint_path_1,weights_only=True))
featureVector_2 = MasterFeatureVector(2)
featureVector_2.load_state_dict(torch.load(feature_vector_ckpt_path_2,weights_only=True))
#Reminder: upsamplingVoxelTransformer resolution is somewhat independent of "grid_size", due to the chunking.
upsampling_network_2 = UpsamplingVoxelTransformer(8)
upsampling_network_2.load_state_dict(torch.load(checkpoint_path_2,weights_only=True))
model_1 = UpsampleShape(
upsampling_network_1,
featureVector_1,
2
)
model_2 = UpsampleShape(
upsampling_network_2,
featureVector_2,
1
)
n_samples = 4
print(len(dataset))
random_indices = torch.randint(0,50,[n_samples])
batch = dataset[list(random_indices)]
result_0 = model_0.sample(n_samples)
model_1.cuda()
result_1 = model_1.sample(low_res_grids.float().cuda())
model_2.cuda()
result_2 = model_2.sample(result_1.float().cuda())
numpy_voxels = result_2[0].detach()
grid_size_low = 8
final_res = 4*grid_size_low
#plot low, intermediate, and high/final resolution
upsampler_0 = nn.Upsample(scale_factor = 4)
upsampler_1 = nn.Upsample(scale_factor = 2)
for i in range(n_samples):
level_0 = upsampler_0.forward(result_0[i,:,:,:].cpu().float().unsqueeze(0).unsqueeze(0)).squeeze(0).squeeze(0)
level_1 = upsampler_1.forward(result_1[i,:,:,:].cpu().float().unsqueeze(0).unsqueeze(0)).squeeze(0).squeeze(0)
level_2 = result_2[i,:,:,:].cpu()
combined = torch.cat((level_0,level_1,level_2,torch.zeros((final_res,final_res,final_res))),dim=2)
plot = k3d.plot()
plot += k3d.voxels(numpy.ones([grid_size_low,grid_size_low,grid_size_low]),opacity=0.01)
plot += k3d.voxels(combined.cpu())
60
0
1.5
plot.display()
```
......
codewords.py
+ 59
− 40
View file @ d8cbe275
......@@ -6,21 +6,19 @@ from dataLoading import dataset
def chunk_index_tensor():
return torch.unflatten(2**torch.arange(0,8),0,(2,2,2))
return (2**torch.arange(0,8))
"""
Non-learnable, non-differentiable Model to generate upscaling target grids.
In the target voxel grid, each voxel is assigned a token 0,...,255.
This will downscale the input grid by a factor of 2.
If we want to use the BCE loss,
we will have to arrange the grid in this shape (*, grid_size,grid_size,grid_size,8)
Instead of (*, grid_size,grid_size,grid_size)
"""
class GenerateTokenGrid(nn.Module):
class GenerateBinaryTokenGrid(nn.Module):
def __init__(self):
super().__init__()
index_tensor = chunk_index_tensor()
self.chunk_index_tensor = index_tensor.unsqueeze(0).unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
#now it has shape (1,2,2,2,1,1,1)
def forward(self,x):
grid_size = x.shape[-1]
......@@ -32,54 +30,56 @@ class GenerateTokenGrid(nn.Module):
chunks = torch.cat(torch.chunk(chunks, factor,-6),-3)
chunks = torch.cat(torch.chunk(chunks, factor,-5),-2)
chunks = torch.cat(torch.chunk(chunks, factor,-4),-1)
#now the dimensions -6,-5,-4 are the dimensions inside each chunk.
weighted_chunks = chunks * self.chunk_index_tensor
#now we just have to sum over the dimensions -6,-5,-4.
res = weighted_chunks.sum(dim=(-6,-5,-4))
return res
#each coordinate inside a chunk must be assigned a fixed power of 2.
#we will broadcast a tensor over the chunks.
#the input can be mapped to the target, using strided convolutions.
#stride =2, ->we have 256 filters, representing ways to fill a 2x2x2 grid.
res = chunks.flatten(-6,-4)
res = res.permute((0,2,3,4,1))
return res
#instead of summing over the chunks we will now just flatten the chunk dimensions.
"""
If we want to use the BCE loss,
we will have to arrange the grid in this shape (*, grid_size,grid_size,grid_size,8)
Instead of (*, grid_size,grid_size,grid_size)
Non-learnable, non-differentiable Model to generate upscaling target grids.
In the target voxel grid, each voxel is assigned a token 0,...,255.
This will downscale the input grid by a factor of 2.
"""
class GenerateBinaryTokenGrid(GenerateTokenGrid):
class GenerateTokenGrid(nn.Module):
def __init__(self):
super().__init__()
index_tensor = chunk_index_tensor()
self.chunk_index_tensor = index_tensor.unsqueeze(0).unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
#now it has shape (1,2,2,2,1,1,1)
self.model = GenerateBinaryTokenGrid()
def forward(self,x):
grid_size = x.shape[-1]
flattened_chunks = self.model.forward(x)
#now the dimensions -6,-5,-4 are the dimensions inside each chunk.
weighted_chunks = flattened_chunks * (2**torch.arange(0,8)).unsqueeze(0).unsqueeze(0).unsqueeze(0).unsqueeze(0)
#now we just have to sum over the dimensions -6,-5,-4.
res = weighted_chunks.sum(dim=(-6,-5,-4))
return res
#each coordinate inside a chunk must be assigned a fixed power of 2.
#we will broadcast a tensor over the chunks.
#the input can be mapped to the target, using strided convolutions.
#stride =2, ->we have 256 filters, representing ways to fill a 2x2x2 grid.
#grid size
factor = grid_size//2
chunks = x.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
chunks = torch.cat(torch.chunk(chunks, factor,-6),-3)
chunks = torch.cat(torch.chunk(chunks, factor,-5),-2)
chunks = torch.cat(torch.chunk(chunks, factor,-4),-1)
res = chunks.flatten(-6,-4)
res = res.permute((0,2,3,4,1))
return res
#instead of summing over the chunks we will now just flatten the chunk dimensions.
"""
Wrapper model, which downscales the "raw" voxel by the specified factor,
......@@ -104,13 +104,14 @@ class TokensToBinary(nn.Module):
def forward(self,x:torch.LongTensor):
index_tensor = chunk_index_tensor().unsqueeze(0).unsqueeze(0).unsqueeze(0).unsqueeze(0).cuda()
#index tensor shape: (1,1,1,1,8)
#index tensor shape:
#(1,1,1,1,2,2,2)
chunks = x.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).cuda()
#chunks shape:
#(batch_dim,2,2,2,1,1,1)
#(batch_dim,g,g,g)
#we need to broadcast the chunks modulo chunk_tensor over the dimensions: -6,-5,-4
#we need to broadcast the chunks modulo chunk_tensor over the dimensions: -1
remainder_tensor = torch.divide(chunks,index_tensor,rounding_mode="floor")
remainder_tensor = torch.remainder(remainder_tensor,2)
#now we know for each position the 1.0 or 0.0
......@@ -130,4 +131,22 @@ class TokensToBinary(nn.Module):
return remainder_tensor
"""
This is used during inference, to map the output distribution ({0,1,...,7}->{0,1})
to the token, so that this diagram commutes:
binary grid ----- GenerateTargetGrid -------------------
| |
| |
GenerateBinaryTokenGrid |
| |
| |
V V
flattened chunk grid -----distribution_to_tokens----> token grid
"""
def distribution_to_tokens(x:torch.Tensor):
x = x.unflatten(-1,(2,2,2)) * chunk_index_tensor().unsqueeze(0).cuda()
x = torch.sum(x,dim=(-3,-2,-1)).unsqueeze(-1)
return x
upsampling_training.py
+ 12
− 3
View file @ d8cbe275
......@@ -39,6 +39,7 @@ def train_layer(
lr:float,
num_epochs:int,
use_bce_loss:bool,
clear_plots:bool,
previous_ckpt = None,
previous_feature_vector = None,
batch_size = 16
......@@ -283,15 +284,19 @@ def train_layer(
plt.yscale("log")
plt.savefig("hierarchicalTransformer/training_loss_curve_"+save_title+".png")
plt.show()
if clear_plots:
plt.close()
plt.plot(train_accuracy_values,label="argmax")
plt.plot(train_soft_accuracy_values,label="softmax correct predictions")
#plt.plot(train_soft_accuracy_values,label="softmax correct predictions")
plt.plot(train_probability_values,label="softmax probability measure")
plt.xlabel("epoch")
plt.ylabel("accuracy (on training set)")
plt.legend()
plt.savefig("hierarchicalTransformer/training_accuracy_"+save_title+".png")
plt.show()
if clear_plots:
plt.close()
plt.plot(validation_losses)
plt.xlabel("epoch")
......@@ -299,12 +304,16 @@ def train_layer(
plt.yscale("log")
plt.savefig("hierarchicalTransformer/validation_loss_curve_"+save_title+".png")
plt.show()
if clear_plots:
plt.close()
plt.plot(validation_accuracy_values,label="argmax")
plt.plot(validation_soft_accuracy_values,label="softmax correct predictions")
#plt.plot(validation_soft_accuracy_values,label="softmax correct predictions")
plt.plot(validation_probability_values,label="softmax probability measure")
plt.xlabel("epoch")
plt.ylabel("accuracy (on validation set)")
plt.legend()
plt.savefig("hierarchicalTransformer/validation_accuracy_"+save_title+".png")
plt.show()
if clear_plots:
plt.close()
\ No newline at end of file
upsampling_transformer.py
+ 1
− 0
View file @ d8cbe275
......@@ -285,6 +285,7 @@ class UpsamplingVoxelTransformer(nn.Module):
next_tokens = next_tokens.unflatten(-1,(2,2,2)) * chunk_index_tensor().unsqueeze(0).cuda()
next_tokens = torch.sum(next_tokens,dim=(-3,-2,-1)).unsqueeze(-1)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment