ref: 98da33500966c6ba2b7b70a7259a33c23abe6b8c
parent: c7b6935bf2bb6287ceca3b3977618c03d1c007d1
author: Jean-Marc Valin <jmvalin@amazon.com>
date: Sun May 21 23:32:53 EDT 2023
Convert PLC weights to blob format
--- a/dnn/lpcnet_plc.c
+++ b/dnn/lpcnet_plc.c
@@ -44,6 +44,7 @@
}
LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) {+ int ret;
RNN_CLEAR(st, 1);
lpcnet_init(&st->lpcnet);
lpcnet_encoder_init(&st->enc);
@@ -67,7 +68,9 @@
return -1;
}
st->remove_dc = !!(options&LPCNET_PLC_DC_FILTER);
- return 0;
+ ret = init_plc_model(&st->model, lpcnet_plc_arrays);
+ celt_assert(ret == 0);
+ return ret;
}
LPCNET_EXPORT LPCNetPLCState *lpcnet_plc_create(int options) {@@ -105,13 +108,14 @@
}
-static void compute_plc_pred(PLCNetState *net, float *out, const float *in) {+static void compute_plc_pred(LPCNetPLCState *st, float *out, const float *in) { float zeros[3*PLC_MAX_RNN_NEURONS] = {0};float dense_out[PLC_DENSE1_OUT_SIZE];
- _lpcnet_compute_dense(&plc_dense1, dense_out, in);
- compute_gruB(&plc_gru1, zeros, net->plc_gru1_state, dense_out);
- compute_gruB(&plc_gru2, zeros, net->plc_gru2_state, net->plc_gru1_state);
- _lpcnet_compute_dense(&plc_out, out, net->plc_gru2_state);
+ PLCNetState *net = &st->plc_net;
+ _lpcnet_compute_dense(&st->model.plc_dense1, dense_out, in);
+ compute_gruB(&st->model.plc_gru1, zeros, net->plc_gru1_state, dense_out);
+ compute_gruB(&st->model.plc_gru2, zeros, net->plc_gru2_state, net->plc_gru1_state);
+ _lpcnet_compute_dense(&st->model.plc_out, out, net->plc_gru2_state);
/* Artificially boost the correlation to make harmonics cleaner. */
out[19] = MIN16(.5f, out[19]+.1f);
}
@@ -127,11 +131,11 @@
/* Update PLC state using FEC, so without Burg features. */
RNN_COPY(&plc_features[2*NB_BANDS], out, NB_FEATURES);
plc_features[2*NB_BANDS+NB_FEATURES] = -1;
- compute_plc_pred(&st->plc_net, discard, plc_features);
+ compute_plc_pred(st, discard, plc_features);
return 1;
} else { float zeros[2*NB_BANDS+NB_FEATURES+1] = {0};- compute_plc_pred(&st->plc_net, out, zeros);
+ compute_plc_pred(st, out, zeros);
if (st->fec_skip > 0) st->fec_skip--;
return 0;
}
@@ -187,7 +191,7 @@
if (st->enable_blending) {LPCNetState copy;
st->plc_net = st->plc_copy[FEATURES_DELAY];
- compute_plc_pred(&st->plc_net, st->features, zeros);
+ compute_plc_pred(st, st->features, zeros);
for (i=0;i<FEATURES_DELAY;i++) {/* FIXME: backtrack state, replace features. */
run_frame_network_deferred(&st->lpcnet, st->features);
@@ -227,7 +231,7 @@
if (!st->blend) {RNN_COPY(&plc_features[2*NB_BANDS], st->enc.features[0], NB_FEATURES);
plc_features[2*NB_BANDS+NB_FEATURES] = 1;
- compute_plc_pred(&st->plc_net, st->features, plc_features);
+ compute_plc_pred(st, st->features, plc_features);
/* Discard an FEC frame that we know we will no longer need. */
if (st->fec_skip) st->fec_skip--;
else if (st->fec_read_pos < st->fec_fill_pos) st->fec_read_pos++;
@@ -352,7 +356,7 @@
float zeros[2*NB_BANDS+NB_FEATURES+1] = {0};RNN_COPY(zeros, plc_features, 2*NB_BANDS);
zeros[2*NB_BANDS+NB_FEATURES] = 1;
- compute_plc_pred(&st->plc_net, st->features, zeros);
+ compute_plc_pred(st, st->features, zeros);
copy = st->lpcnet;
lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, 0);
/* Undo initial DC offset removal so that we can take into account the last 5ms of synthesis. */
@@ -405,7 +409,7 @@
if (st->loss_count == 0) {RNN_COPY(&plc_features[2*NB_BANDS], st->enc.features[0], NB_FEATURES);
plc_features[2*NB_BANDS+NB_FEATURES] = 1;
- compute_plc_pred(&st->plc_net, st->features, plc_features);
+ compute_plc_pred(st, st->features, plc_features);
lpcnet_synthesize_impl(&st->lpcnet, st->enc.features[0], &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET);
lpcnet_synthesize_tail_impl(&st->lpcnet, pcm, FRAME_SIZE-TRAINING_OFFSET, FRAME_SIZE-TRAINING_OFFSET);
}
@@ -428,7 +432,7 @@
process_queued_update(st);
st->enc.pcount = 0;
- compute_plc_pred(&st->plc_net, st->features, zeros);
+ compute_plc_pred(st, st->features, zeros);
if (st->loss_count >= 10) st->features[0] = MAX16(-10, st->features[0]+att_table[9] - 2*(st->loss_count-9));
else st->features[0] = MAX16(-10, st->features[0]+att_table[st->loss_count]);
--- a/dnn/lpcnet_private.h
+++ b/dnn/lpcnet_private.h
@@ -98,6 +98,7 @@
short dc_buf[TRAINING_OFFSET];
int queued_update;
short queued_samples[FRAME_SIZE];
+ PLCModel model;
};
extern float ceps_codebook1[];
--- a/dnn/nnet.h
+++ b/dnn/nnet.h
@@ -147,7 +147,7 @@
extern const WeightArray lpcnet_arrays[];
-
+extern const WeightArray lpcnet_plc_arrays[];
int mdense_init(MDenseLayer *layer, const WeightArray *arrays,
const char *bias,
--- a/dnn/training_tf2/dump_plc.py
+++ b/dnn/training_tf2/dump_plc.py
@@ -27,6 +27,7 @@
'''
import lpcnet_plc
+import io
import sys
import numpy as np
from tensorflow.keras.optimizers import Adam
@@ -41,11 +42,17 @@
max_conv_inputs = 1
def printVector(f, vector, name, dtype='float', dotp=False):
+ global array_list
if dotp:
vector = vector.reshape((vector.shape[0]//4, 4, vector.shape[1]//8, 8))
vector = vector.transpose((2, 0, 3, 1))
v = np.reshape(vector, (-1));
#print('static const float ', name, '[', len(v), '] = \n', file=f)+ if name not in array_list:
+ array_list.append(name)
+ f.write('#ifndef USE_WEIGHTS_FILE\n')+ f.write('#define WEIGHTS_{}_DEFINED\n'.format(name))+ f.write('#define WEIGHTS_{}_TYPE WEIGHT_TYPE_{}\n'.format(name, dtype)) f.write('static const {} {}[{}] = {{\n '.format(dtype, name, len(v)))for i in range(0, len(v)):
f.write('{}'.format(v[i]))@@ -58,7 +65,8 @@
else:
f.write(" ")#print(v, file=f)
- f.write('\n};\n\n')+ f.write('\n};\n')+ f.write('#endif\n\n')return;
def printSparseVector(f, A, name, have_diag=True):
@@ -122,11 +130,11 @@
reset_after = 1
neurons = weights[0].shape[1]//3
max_rnn_neurons = max(max_rnn_neurons, neurons)
- f.write('const SparseGRULayer {} = {{\n {}_bias,\n {}_subias,\n {}_recurrent_weights_diag,\n {}_recurrent_weights,\n {}_recurrent_weights_idx,\n {}, ACTIVATION_{}, {}\n}};\n\n'- .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after))
hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3)) hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))- hf.write('extern const SparseGRULayer {};\n\n'.format(name));+ model_struct.write(' SparseGRULayer {};\n'.format(name));+ model_init.write(' if (sparse_gru_init(&model->{}, arrays, "{}_bias", "{}_subias", "{}_recurrent_weights_diag", "{}_recurrent_weights", "{}_recurrent_weights_idx", {}, ACTIVATION_{}, {})) return 1;\n'+ .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after))
return True
def dump_gru_layer(self, f, hf):
@@ -158,11 +166,11 @@
reset_after = 1
neurons = weights[0].shape[1]//3
max_rnn_neurons = max(max_rnn_neurons, neurons)
- f.write('const GRULayer {} = {{\n {}_bias,\n {}_subias,\n {}_weights,\n {}_weights_idx,\n {}_recurrent_weights,\n {}, {}, ACTIVATION_{}, {}\n}};\n\n'- .format(name, name, name, name, name, name, weights[0].shape[0], weights[0].shape[1]//3, activation, reset_after))
hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3)) hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))- hf.write('extern const GRULayer {};\n\n'.format(name));+ model_struct.write(' GRULayer {};\n'.format(name));+ model_init.write(' if (gru_init(&model->{}, arrays, "{}_bias", "{}_subias", "{}_weights", "{}_weights_idx", "{}_recurrent_weights", {}, {}, ACTIVATION_{}, {})) return 1;\n'+ .format(name, name, name, name, name, name, weights[0].shape[0], weights[0].shape[1]//3, activation, reset_after))
return True
GRU.dump_layer = dump_gru_layer
@@ -178,10 +186,10 @@
def dump_dense_layer_impl(name, weights, bias, activation, f, hf):
printVector(f, weights, name + '_weights')
printVector(f, bias, name + '_bias')
- f.write('const DenseLayer {} = {{\n {}_bias,\n {}_weights,\n {}, {}, ACTIVATION_{}\n}};\n\n'- .format(name, name, name, weights.shape[0], weights.shape[1], activation))
hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights.shape[1]))- hf.write('extern const DenseLayer {};\n\n'.format(name));+ model_struct.write(' DenseLayer {};\n'.format(name));+ model_init.write(' if (dense_init(&model->{}, arrays, "{}_bias", "{}_weights", {}, {}, ACTIVATION_{})) return 1;\n'+ .format(name, name, name, weights.shape[0], weights.shape[1], activation))
def dump_dense_layer(self, f, hf):
name = self.name
@@ -202,12 +210,12 @@
printVector(f, weights[-1], name + '_bias')
activation = self.activation.__name__.upper()
max_conv_inputs = max(max_conv_inputs, weights[0].shape[1]*weights[0].shape[0])
- f.write('const Conv1DLayer {} = {{\n {}_bias,\n {}_weights,\n {}, {}, {}, ACTIVATION_{}\n}};\n\n'- .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation))
hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[2])) hf.write('#define {}_STATE_SIZE ({}*{})\n'.format(name.upper(), weights[0].shape[1], (weights[0].shape[0]-1))) hf.write('#define {}_DELAY {}\n'.format(name.upper(), (weights[0].shape[0]-1)//2))- hf.write('extern const Conv1DLayer {};\n\n'.format(name));+ model_struct.write(' Conv1DLayer {};\n'.format(name));+ model_init.write(' if (conv1d_init(&model->{}, arrays, "{}_bias", "{}_weights", {}, {}, {}, ACTIVATION_{})) return 1;\n'+ .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation))
return True
Conv1D.dump_layer = dump_conv1d_layer
@@ -235,6 +243,12 @@
f = open(cfile, 'w')
hf = open(hfile, 'w')
+model_struct = io.StringIO()
+model_init = io.StringIO()
+model_struct.write('typedef struct {\n')+model_init.write('#ifndef DUMP_BINARY_WEIGHTS\n')+model_init.write('int init_plc_model(PLCModel *model, const WeightArray *arrays) {\n')+array_list = []
f.write('/*This file is automatically generated from a Keras model*/\n')@@ -250,7 +264,20 @@
layer_list.append(layer.name)
#dump_sparse_gru(model.get_layer('gru_a'), f, hf)+f.write('#ifndef USE_WEIGHTS_FILE\n')+f.write('const WeightArray lpcnet_plc_arrays[] = {\n')+for name in array_list:
+ f.write('#ifdef WEIGHTS_{}_DEFINED\n'.format(name))+ f.write(' {{"{}", WEIGHTS_{}_TYPE, sizeof({}), {}}},\n'.format(name, name, name, name))+ f.write('#endif\n')+f.write(' {NULL, 0, 0, NULL}\n};\n')+f.write('#endif\n')+model_init.write(' return 0;\n}\n')+model_init.write('#endif\n')+f.write(model_init.getvalue())
+
+
hf.write('#define PLC_MAX_RNN_NEURONS {}\n\n'.format(max_rnn_neurons)) #hf.write('#define PLC_MAX_CONV_INPUTS {}\n\n'.format(max_conv_inputs))@@ -257,7 +284,11 @@
hf.write('typedef struct {\n')for i, name in enumerate(layer_list):
hf.write(' float {}_state[{}_STATE_SIZE];\n'.format(name, name.upper())) -hf.write('} PLCNetState;\n')+hf.write('} PLCNetState;\n\n')+
+model_struct.write('} PLCModel;\n\n')+hf.write(model_struct.getvalue())
+hf.write('int init_plc_model(PLCModel *model, const WeightArray *arrays);\n\n') hf.write('\n\n#endif\n')--
⑨