From d50125c95f5451139b27338299e09da5bd09263b Mon Sep 17 00:00:00 2001 From: Andrew Kane Date: Tue, 24 Sep 2024 00:22:13 -0700 Subject: [PATCH] Fixed rounding [skip ci] --- src/minivec.h | 4 ++-- test/expected/minivec.out | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/minivec.h b/src/minivec.h index 8a57b0f..b4eafec 100644 --- a/src/minivec.h +++ b/src/minivec.h @@ -103,7 +103,7 @@ Float4ToFp8Unchecked(float num) int s; exponent -= 127; - s = mantissa & 0x001FFFFF; + s = mantissa & 0x000FFFFF; /* Subnormal */ if (exponent < -14) @@ -112,7 +112,7 @@ Float4ToFp8Unchecked(float num) mantissa >>= diff; mantissa += 1 << (23 - diff); - s |= mantissa & 0x001FFFFF; + s |= mantissa & 0x000FFFFF; } m = mantissa >> 21; diff --git a/test/expected/minivec.out b/test/expected/minivec.out index bda451a..7b9bc4e 100644 --- a/test/expected/minivec.out +++ b/test/expected/minivec.out @@ -165,7 +165,7 @@ ERROR: expected 2 dimensions, not 3 SELECT '[1,2,3]'::minivec + '[4,5,6]'; ?column? ---------- - [5,7,10] + [5,7,8] (1 row) SELECT '[61439]'::minivec + '[61439]'; @@ -189,7 +189,7 @@ ERROR: different minivec dimensions 2 and 1 SELECT '[1,2,3]'::minivec * '[4,5,6]'; ?column? ----------- - [4,10,20] + [4,10,16] (1 row) SELECT '[61439]'::minivec * '[61439]'; @@ -403,7 +403,7 @@ SELECT inner_product('[448]'::minivec, '[448]'); SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]'); inner_product --------------- - 46 + 44 (1 row) SELECT '[1,2]'::minivec <#> '[3,4]'; @@ -497,7 +497,7 @@ SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]'); SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]'); l1_distance ------------- - 11 + 7 (1 row) SELECT '[0,0]'::minivec <+> '[3,4]';