Commit ca783c0e authored by Benjamin Vandersmissen's avatar Benjamin Vandersmissen
Browse files

Added the divide-by-sum importance measure and removed the normalize-by-std importance measure.

parent 182b01e9
import torch
import copy
import math
import os
import numpy as np
......@@ -27,15 +25,8 @@ def softmax(inp):
return torch.softmax(inp, dim=-1)
def normalize_std(inp):
Divides by std of the initialization,
but because the std is only variable in the term math.sqrt(1/float(fan_in + fan_out)),
we divide by that instead
# TODO: this is only for GLOROT UNIFORM, implement the others when needed
fan_in, fan_out = torch.nn.init._calculate_fan_in_and_fan_out(inp)
return inp / math.sqrt(1 / float(fan_in + fan_out))
def normalize_sum(inp):
return inp/inp.sum()
def prune_by_magnitude(model: torch.nn.Module, percentage=0.2, transformation=identity):
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment