Commit 01abee2b authored by Bartosz Podrygajlo's avatar Bartosz Podrygajlo

Reduce rounding error in rshift of sum_det

 - Use a uint64_t
 - move shift after the sum
 - add bias before shifting to reduce rounding error
parent 9f03fde6
......@@ -493,10 +493,12 @@ static simde__m128i nr_ulsch_comp_muli_sum(simde__m128i input_x,
//print_ints("rx_re:",(int32_t*)&xy_re_128[0]);
//print_ints("rx_Img:",(int32_t*)&xy_im_128[0]);
//divide by matrix det and convert back to Q15 before packing
uint32_t sum_det = 0;
uint64_t sum_det = 0;
for (int k = 0; k < 4; k++) {
sum_det += (((uint32_t *)&det)[k]) >> 2;
sum_det += (((uint32_t *)&det)[k]);
}
// Add bias to reduce rounding error
sum_det = (sum_det + 2) >> 2;
int b = log2_approx(sum_det) - 8;
if (b > 0) {
......@@ -944,10 +946,12 @@ static uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms,
// Magnitude computation
if (mod_order > 2) {
uint32_t sum_det = 0;
uint64_t sum_det = 0;
for (int k = 0; k < 4; k++) {
sum_det += (((uint32_t *)&determ_fin_128[0])[k]) >> 2;
sum_det += (((uint32_t *)&determ_fin_128[0])[k]);
}
// Add bias to reduce rounding error
sum_det = (sum_det + 2) >> 2;
int b = log2_approx(sum_det) - 8;
if (b > 0) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment