org.deeplearning4j.nn.conf.distribution.Distribution Java Examples

protected void handleWeightInitBackwardCompatibility(BaseLayer baseLayer, ObjectNode on){
    if(on != null && on.has("weightInit") ){
        //Legacy format JSON
        if(on.has("weightInit")){
            String wi = on.get("weightInit").asText();
            try{
                WeightInit w = WeightInit.valueOf(wi);
                Distribution d = null;
                if(w == WeightInit.DISTRIBUTION && on.has("dist")){
                    String dist = on.get("dist").toString();
                    d = NeuralNetConfiguration.mapper().readValue(dist, Distribution.class);
                }
                IWeightInit iwi = w.getWeightInitFunction(d);
                baseLayer.setWeightInitFn(iwi);
            } catch (Throwable t){
                log.warn("Failed to infer weight initialization from legacy JSON format",t);
            }
        }
    }
}
 

@Override
public INDArray getParameter(Layer layer, String paramKey, int iteration, int epoch, boolean train, LayerWorkspaceMgr workspaceMgr) {

    ParamInitializer init = layer.conf().getLayer().initializer();
    INDArray param = layer.getParam(paramKey);
    if (train && init.isWeightParam(layer.conf().getLayer(), paramKey) ||
            (applyToBias && init.isBiasParam(layer.conf().getLayer(), paramKey))) {

        org.nd4j.linalg.api.rng.distribution.Distribution dist = Distributions.createDistribution(distribution);
        INDArray noise = dist.sample(param.ulike());
        INDArray out = workspaceMgr.createUninitialized(ArrayType.INPUT, param.dataType(), param.shape(), param.ordering());

        if (additive) {
            Nd4j.getExecutioner().exec(new AddOp(param, noise,out));
        } else {
            Nd4j.getExecutioner().exec(new MulOp(param, noise, out));
        }
        return out;
    }
    return param;
}
 

private static DenseLayer fullyConnected(String name, int out, double bias, Distribution dist) {
    return new DenseLayer.Builder().name(name)
            .nOut(out)
            .biasInit(bias)
            .dist(dist)
            .build();
}
 

/**
 * @param distribution Distribution for noise
 * @param applyToBias  If true: apply to biases also. If false (default): apply only to weights
 * @param additive     If true: noise is added to weights. If false: noise is multiplied by weights
 */
public WeightNoise(@JsonProperty("distribution") Distribution distribution,
                   @JsonProperty("applyToBias") boolean applyToBias,
                   @JsonProperty("additive") boolean additive) {
    this.distribution = distribution;
    this.applyToBias = applyToBias;
    this.additive = additive;
}
 

public GraphBuilder nOutReplace(String layerName, int nOut, WeightInit scheme, Distribution dist) {
    if(scheme == WeightInit.DISTRIBUTION) {
        throw new UnsupportedOperationException("Not supported!, Use " +
                "nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext)) instead!");
    }
    return nOutReplace(layerName, nOut, scheme.getWeightInitFunction(), new WeightInitDistribution(dist));
}
 

public GraphBuilder nOutReplace(String layerName, int nOut, Distribution dist, WeightInit scheme) {
    if(scheme == WeightInit.DISTRIBUTION) {
        throw new UnsupportedOperationException("Not supported!, Use " +
                "nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext)) instead!");
    }
    return nOutReplace(layerName, nOut, new WeightInitDistribution(dist), scheme.getWeightInitFunction());
}
 

@Override
public INDArray init(double fanIn, double fanOut, long[] shape, char order, INDArray paramView) {
    //org.nd4j.linalg.api.rng.distribution.Distribution not serializable
    org.nd4j.linalg.api.rng.distribution.Distribution dist = Distributions.createDistribution(distribution);
    if (dist instanceof OrthogonalDistribution) {
        dist.sample(paramView.reshape(order, shape));
    } else {
        dist.sample(paramView);
    }
    return paramView.reshape(order, shape);
}
 

public WeightInitDistribution(@JsonProperty("distribution") Distribution distribution) {
    if(distribution == null) {
        // Would fail later below otherwise
        throw new IllegalArgumentException("Must set distribution!");
    }
    this.distribution = distribution;
}
 

public T weightInit(Distribution distribution){
    weightInit(WeightInit.DISTRIBUTION);
    return dist(distribution);
}
 

private static  DenseLayer fullyConnected(String name, int out, double bias, double dropOut, Distribution dist) {
    return new DenseLayer.Builder().name(name).nOut(out).biasInit(bias).dropOut(dropOut).dist(dist).build();
}
 

/**
 * Handle {@link WeightInit} and {@link Distribution} from legacy configs in Json format. Copied from handling of {@link Activation}
 * above.
 * @return True if all is well and layer iteration shall continue. False else-wise.
 */
private static void handleLegacyWeightInitFromJson(String json, Layer layer, ObjectMapper mapper, JsonNode vertices) {
    if (layer instanceof BaseLayer && ((BaseLayer) layer).getWeightInitFn() == null) {
        String layerName = layer.getLayerName();

        try {
            if (vertices == null) {
                JsonNode jsonNode = mapper.readTree(json);
                vertices = jsonNode.get("vertices");
            }

            JsonNode vertexNode = vertices.get(layerName);
            JsonNode layerVertexNode = vertexNode.get("LayerVertex");
            if (layerVertexNode == null || !layerVertexNode.has("layerConf")
                    || !layerVertexNode.get("layerConf").has("layer")) {
                return;
            }
            JsonNode layerWrapperNode = layerVertexNode.get("layerConf").get("layer");

            if (layerWrapperNode == null || layerWrapperNode.size() != 1) {
                return;
            }

            JsonNode layerNode = layerWrapperNode.elements().next();
            JsonNode weightInit = layerNode.get("weightInit"); //Should only have 1 element: "dense", "output", etc
            JsonNode distribution = layerNode.get("dist");

            Distribution dist = null;
            if(distribution != null) {
                dist = mapper.treeToValue(distribution, Distribution.class);
            }

            if (weightInit != null) {
                final IWeightInit wi = WeightInit.valueOf(weightInit.asText()).getWeightInitFunction(dist);
                ((BaseLayer) layer).setWeightInitFn(wi);
            }

        } catch (IOException e) {
            log.warn("Layer with null ActivationFn field or pre-0.7.2 activation function detected: could not parse JSON",
                    e);
        }
    }
}
 

/**
 * @param distribution Distribution for additive noise
 */
public WeightNoise(Distribution distribution) {
    this(distribution, false, true);
}
 

/**
 * Handle {@link WeightInit} and {@link Distribution} from legacy configs in Json format. Copied from handling of {@link Activation}
 * above.
 * @return True if all is well and layer iteration shall continue. False else-wise.
 */
private static boolean handleLegacyWeightInitFromJson(String json, Layer l, ObjectMapper mapper, JsonNode confs, int layerCount) {
    if ((l instanceof BaseLayer) && ((BaseLayer) l).getWeightInitFn() == null) {
        try {
            JsonNode jsonNode = mapper.readTree(json);
            if (confs == null) {
                confs = jsonNode.get("confs");
            }
            if (confs instanceof ArrayNode) {
                ArrayNode layerConfs = (ArrayNode) confs;
                JsonNode outputLayerNNCNode = layerConfs.get(layerCount);
                if (outputLayerNNCNode == null)
                    return false; //Should never happen...
                JsonNode layerWrapperNode = outputLayerNNCNode.get("layer");

                if (layerWrapperNode == null || layerWrapperNode.size() != 1) {
                    return true;
                }

                JsonNode layerNode = layerWrapperNode.elements().next();
                JsonNode weightInit = layerNode.get("weightInit"); //Should only have 1 element: "dense", "output", etc
                JsonNode distribution = layerNode.get("dist");

                Distribution dist = null;
                if(distribution != null) {
                    dist = mapper.treeToValue(distribution, Distribution.class);
                }

                if (weightInit != null) {
                    final IWeightInit wi = WeightInit.valueOf(weightInit.asText()).getWeightInitFunction(dist);
                    ((BaseLayer) l).setWeightInitFn(wi);
                }
            }

        } catch (IOException e) {
            log.warn("Layer with null WeightInit detected: " + l.getLayerName() + ", could not parse JSON",
                    e);
        }
    }
    return true;

}
 

/**
 * Modify the architecture of a layer by changing nOut
 * Note this will also affect the layer that follows the layer specified, unless it is the output layer
 * Can specify different weight init schemes for the specified layer and the layer that follows it.
 *
 * @param layerNum The index of the layer to change nOut of
 * @param nOut     Value of nOut to change to
 * @param scheme   Weight init scheme to use for params in layerNum
 * @param distNext Distribution to use for parmas in layerNum+1
 * @return Builder
 * @see org.deeplearning4j.nn.weights.WeightInitDistribution
 */
public Builder nOutReplace(int layerNum, int nOut, WeightInit scheme, Distribution distNext) {
    if(scheme == WeightInit.DISTRIBUTION) {
        throw new UnsupportedOperationException("Not supported!, Use " +
                "nOutReplace(int layerNum, int nOut, Distribution dist, Distribution distNext) instead!");
    }
    return nOutReplace(layerNum, nOut, scheme.getWeightInitFunction(), new WeightInitDistribution(distNext));
}
 

/**
 * Set weight initialization scheme to random sampling via the specified distribution.
 * Equivalent to: {@code .weightInit(new WeightInitDistribution(distribution))}
 *
 * @param distribution Distribution to use for weight initialization
 */
public Builder weightInit(Distribution distribution){
    return weightInit(new WeightInitDistribution(distribution));
}
 

/**
 * Modify the architecture of a vertex layer by changing nIn of the specified layer.<br>
 * Note that only the specified layer will be modified - all other layers will not be changed by this call.
 *
 * @param layerName The name of the layer to change nIn of
 * @param nIn       Value of nIn to change to
 * @param scheme    Weight init scheme to use for params in layerName and the layers following it
 * @return GraphBuilder
 */
public GraphBuilder nInReplace(String layerName, int nIn, WeightInit scheme, Distribution dist) {
    return nInReplace(layerName, nIn, scheme.getWeightInitFunction(dist));
}
 

/**
 * Modified nOut of specified layer. Also affects layers following layerName unless they are output layers
 * @param layerName The name of the layer to change nOut of
 * @param nOut      Value of nOut to change to
 * @param dist      Weight distribution scheme to use for layerName
 * @param distNext  Weight distribution scheme for layers following layerName
 * @return GraphBuilder
 * @see org.deeplearning4j.nn.weights.WeightInit DISTRIBUTION
 */
public GraphBuilder nOutReplace(String layerName, int nOut, Distribution dist, Distribution distNext) {
    return nOutReplace(layerName, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext));
}
 

/**
 * @param distribution Distribution for noise
 * @param additive     If true: noise is added to weights. If false: noise is multiplied by weights
 */
public WeightNoise(Distribution distribution, boolean additive) {
    this(distribution, false, additive);
}
 

/**
 * Modify the architecture of a vertex layer by changing nOut
 * Note this will also affect the vertex layer that follows the layer specified, unless it is the output layer
 * Currently does not support modifying nOut of layers that feed into non-layer vertices like merge, subset etc
 * To modify nOut for such vertices use remove vertex, followed by add vertex
 * Can specify different weight init schemes for the specified layer and the layer that follows it.
 *
 * @param layerName The name of the layer to change nOut of
 * @param nOut      Value of nOut to change to
 * @param dist      Weight distribution scheme to use
 * @return GraphBuilder
 * @see org.deeplearning4j.nn.weights.WeightInit DISTRIBUTION
 */
public GraphBuilder nOutReplace(String layerName, int nOut, Distribution dist) {
    return nOutReplace(layerName, nOut, dist, dist);
}
 

/**
 * Modify the architecture of a vertex layer by changing nIn of the specified layer.<br>
 * Note that only the specified layer will be modified - all other layers will not be changed by this call.
 *
 * @param layerNum The number of the layer to change nIn of
 * @param nIn      Value of nIn to change to
 * @param scheme   Weight init scheme to use for params in layerName
 * @return Builder
 */
public Builder nInReplace(int layerNum, int nIn, WeightInit scheme, Distribution dist) {
    return nInReplace(layerNum, nIn, scheme.getWeightInitFunction(dist));
}
 

/**
 * Set weight initialization scheme to random sampling via the specified distribution.
 * Equivalent to: {@code .weightInit(new WeightInitDistribution(distribution))}
 * Note: values set by this method will be applied to all applicable layers in the network, unless a different
 * value is explicitly set on a given layer. In other words: values set via this method are used as the default
 * value, and can be overridden on a per-layer basis.
 *
 * @param distribution Distribution to use for weight initialization
 */
public Builder weightInit(Distribution distribution){
    return weightInit(new WeightInitDistribution(distribution));
}
 

/**
 * Set the weight initialization for the recurrent weights, based on the specified distribution. Not that if
 * this is not set explicitly, the same weight initialization as the layer input weights is also used for the
 * recurrent weights.
 *
 * @param dist Distribution to use for initializing the recurrent weights
 */
public T weightInitRecurrent(Distribution dist) {
    this.setWeightInitFnRecurrent(new WeightInitDistribution(dist));
    return (T) this;
}
 

/**
 * Set weight initialization scheme to random sampling via the specified distribution. Equivalent to: {@code
 * .weightInit(new WeightInitDistribution(distribution))}
 *
 * @param distribution Distribution to use for weight initialization
 */
public T weightInit(Distribution distribution) {
    return weightInit(new WeightInitDistribution(distribution));
}
 

/**
 * Modify the architecture of a layer by changing nOut
 * Note this will also affect the layer that follows the layer specified, unless it is the output layer
 * Can specify different weight init schemes for the specified layer and the layer that follows it.
 *
 * @param layerNum   The index of the layer to change nOut of
 * @param nOut       Value of nOut to change to
 * @param dist       Distribution to use for parmas in layerNum
 * @param schemeNext Weight init scheme to use for params in layerNum+1
 * @return Builder
 * @see org.deeplearning4j.nn.weights.WeightInitDistribution
 */
public Builder nOutReplace(int layerNum, int nOut, Distribution dist, WeightInit schemeNext) {
    return nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), schemeNext.getWeightInitFunction());
}
 

/**
 * Modify the architecture of a layer by changing nOut
 * Note this will also affect the layer that follows the layer specified, unless it is the output layer
 * Can specify different weight init schemes for the specified layer and the layer that follows it.
 *
 * @param layerNum The index of the layer to change nOut of
 * @param nOut     Value of nOut to change to
 * @param dist     Distribution to use for params in the layerNum
 * @param distNext Distribution to use for parmas in layerNum+1
 * @return Builder
 * @see org.deeplearning4j.nn.weights.WeightInitDistribution
 */
public Builder nOutReplace(int layerNum, int nOut, Distribution dist, Distribution distNext) {
    return nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext));
}
 

/**
 * Modify the architecture of a layer by changing nOut
 * Note this will also affect the layer that follows the layer specified, unless it is the output layer
 *
 * @param layerNum The index of the layer to change nOut of
 * @param nOut     Value of nOut to change to
 * @param dist     Distribution to use in conjunction with weight init DISTRIBUTION for params in layernum and layernum+1
 * @return Builder
 * @see org.deeplearning4j.nn.weights.WeightInit DISTRIBUTION
 */
public Builder nOutReplace(int layerNum, int nOut, Distribution dist) {
    return nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(dist));
}