Fixed rounding [skip ci]

This commit is contained in:
Andrew Kane
2024-09-24 00:22:13 -07:00
parent d805378471
commit d50125c95f
2 changed files with 6 additions and 6 deletions

View File

@@ -103,7 +103,7 @@ Float4ToFp8Unchecked(float num)
int s; int s;
exponent -= 127; exponent -= 127;
s = mantissa & 0x001FFFFF; s = mantissa & 0x000FFFFF;
/* Subnormal */ /* Subnormal */
if (exponent < -14) if (exponent < -14)
@@ -112,7 +112,7 @@ Float4ToFp8Unchecked(float num)
mantissa >>= diff; mantissa >>= diff;
mantissa += 1 << (23 - diff); mantissa += 1 << (23 - diff);
s |= mantissa & 0x001FFFFF; s |= mantissa & 0x000FFFFF;
} }
m = mantissa >> 21; m = mantissa >> 21;

View File

@@ -165,7 +165,7 @@ ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::minivec + '[4,5,6]'; SELECT '[1,2,3]'::minivec + '[4,5,6]';
?column? ?column?
---------- ----------
[5,7,10] [5,7,8]
(1 row) (1 row)
SELECT '[61439]'::minivec + '[61439]'; SELECT '[61439]'::minivec + '[61439]';
@@ -189,7 +189,7 @@ ERROR: different minivec dimensions 2 and 1
SELECT '[1,2,3]'::minivec * '[4,5,6]'; SELECT '[1,2,3]'::minivec * '[4,5,6]';
?column? ?column?
----------- -----------
[4,10,20] [4,10,16]
(1 row) (1 row)
SELECT '[61439]'::minivec * '[61439]'; SELECT '[61439]'::minivec * '[61439]';
@@ -403,7 +403,7 @@ SELECT inner_product('[448]'::minivec, '[448]');
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]'); SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]');
inner_product inner_product
--------------- ---------------
46 44
(1 row) (1 row)
SELECT '[1,2]'::minivec <#> '[3,4]'; SELECT '[1,2]'::minivec <#> '[3,4]';
@@ -497,7 +497,7 @@ SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]'); SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]');
l1_distance l1_distance
------------- -------------
11 7
(1 row) (1 row)
SELECT '[0,0]'::minivec <+> '[3,4]'; SELECT '[0,0]'::minivec <+> '[3,4]';