From 29380ff77d5da51d3065b810a9f93171f8724c1a Mon Sep 17 00:00:00 2001 From: filipe oliveira Date: Sat, 15 Oct 2022 10:17:41 +0100 Subject: [PATCH] optimizing d2string() and addReplyDouble() with grisu2: double to string conversion based on Florian Loitsch's Grisu-algorithm (#10587) All commands / use cases that heavily rely on double to a string representation conversion, (e.g. meaning take a double-precision floating-point number like 1.5 and return a string like "1.5" ), could benefit from a performance boost by swapping snprintf(buf,len,"%.17g",value) by the equivalent [fpconv_dtoa](https://github.com/night-shift/fpconv) or any other algorithm that ensures 100% coverage of conversion. This is a well-studied topic and Projects like MongoDB. RedPanda, PyTorch leverage libraries ( fmtlib ) that use the optimized double to string conversion underneath. The positive impact can be substantial. This PR uses the grisu2 approach ( grisu explained on https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf section 5 ). test suite changes: Despite being compatible, in some cases it produces a different result from printf, and some tests had to be adjusted. one case is that `%.17g` (which means %e or %f which ever is shorter), chose to use `5000000000` instead of 5e+9, which sounds like a bug? In other cases, we changed TCL to compare numbers instead of strings to ignore minor rounding issues (`expr 0.8 == 0.79999999999999999`) --- .gitignore | 1 + deps/Makefile | 7 + deps/fpconv/LICENSE.txt | 23 +++ deps/fpconv/Makefile | 27 +++ deps/fpconv/README.md | 9 + deps/fpconv/fpconv_dtoa.c | 373 ++++++++++++++++++++++++++++++++++++ deps/fpconv/fpconv_dtoa.h | 45 +++++ deps/fpconv/fpconv_powers.h | 133 +++++++++++++ src/Makefile | 6 +- src/debug.c | 9 +- src/networking.c | 12 +- src/rdb.c | 7 +- src/rio.c | 5 +- src/script_lua.c | 5 +- src/util.c | 7 +- src/util.h | 2 +- tests/integration/rdb.tcl | 2 +- tests/unit/type/zset.tcl | 12 +- 18 files changed, 662 insertions(+), 23 deletions(-) create mode 100644 deps/fpconv/LICENSE.txt create mode 100644 deps/fpconv/Makefile create mode 100644 deps/fpconv/README.md create mode 100644 deps/fpconv/fpconv_dtoa.c create mode 100644 deps/fpconv/fpconv_dtoa.h create mode 100644 deps/fpconv/fpconv_powers.h diff --git a/.gitignore b/.gitignore index e03c834d0..5ed94f1da 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ deps/lua/src/lua deps/lua/src/luac deps/lua/src/liblua.a deps/hdr_histogram/libhdrhistogram.a +deps/fpconv/libfpconv.a tests/tls/* .make-* .prerequisites diff --git a/deps/Makefile b/deps/Makefile index 96dbb8c1d..c0dc71120 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -40,6 +40,7 @@ distclean: -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true + -(cd fpconv && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) .PHONY: distclean @@ -66,6 +67,12 @@ hdr_histogram: .make-prerequisites .PHONY: hdr_histogram +fpconv: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd fpconv && $(MAKE) + +.PHONY: fpconv + ifeq ($(uname_S),SunOS) # Make isinf() available LUA_CFLAGS= -D__C99FEATURES__=1 diff --git a/deps/fpconv/LICENSE.txt b/deps/fpconv/LICENSE.txt new file mode 100644 index 000000000..36b7cd93c --- /dev/null +++ b/deps/fpconv/LICENSE.txt @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/deps/fpconv/Makefile b/deps/fpconv/Makefile new file mode 100644 index 000000000..2654888e4 --- /dev/null +++ b/deps/fpconv/Makefile @@ -0,0 +1,27 @@ +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +AR= ar +ARFLAGS= rcs + +libfpconv.a: fpconv_dtoa.o + $(AR) $(ARFLAGS) $@ $+ + +fpconv_dtoa.o: fpconv_dtoa.h fpconv_dtoa.c + +.c.o: + $(R_CC) -c $< + +clean: + rm -f *.o + rm -f *.a + + diff --git a/deps/fpconv/README.md b/deps/fpconv/README.md new file mode 100644 index 000000000..d64ca685a --- /dev/null +++ b/deps/fpconv/README.md @@ -0,0 +1,9 @@ +libfpconv + +---------------------------------------------- + +Fast and accurate double to string conversion based on Florian Loitsch's Grisu-algorithm[1]. + +This port contains a subset of the 'C' version of Fast and accurate double to string conversion based on Florian Loitsch's Grisu-algorithm available at [github.com/night-shift/fpconv](https://github.com/night-shift/fpconv)). + +[1] https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf diff --git a/deps/fpconv/fpconv_dtoa.c b/deps/fpconv/fpconv_dtoa.c new file mode 100644 index 000000000..ad2f2de74 --- /dev/null +++ b/deps/fpconv/fpconv_dtoa.c @@ -0,0 +1,373 @@ +/* fpconv_dtoa.c -- floating point conversion utilities. + * + * Fast and accurate double to string conversion based on Florian Loitsch's + * Grisu-algorithm[1]. + * + * [1] https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2013-2019, night-shift + * Copyright (c) 2009, Florian Loitsch < florian.loitsch at inria dot fr > + * All rights reserved. + * + * Boost Software License - Version 1.0 - August 17th, 2003 + * + * Permission is hereby granted, free of charge, to any person or organization + * obtaining a copy of the software and accompanying documentation covered by + * this license (the "Software") to use, reproduce, display, distribute, + * execute, and transmit the Software, and to prepare derivative works of the + * Software, and to permit third-parties to whom the Software is furnished to + * do so, all subject to the following: + * + * The copyright notices in the Software and this entire statement, including + * the above license grant, this restriction and the following disclaimer, + * must be included in all copies of the Software, in whole or in part, and + * all derivative works of the Software, unless such copies or derivative + * works are solely in the form of machine-executable object code generated by + * a source language processor. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT + * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE + * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "fpconv_dtoa.h" + +#include "fpconv_powers.h" + +#include +#include + +#define fracmask 0x000FFFFFFFFFFFFFU +#define expmask 0x7FF0000000000000U +#define hiddenbit 0x0010000000000000U +#define signmask 0x8000000000000000U +#define expbias (1023 + 52) + +#define absv(n) ((n) < 0 ? -(n) : (n)) +#define minv(a, b) ((a) < (b) ? (a) : (b)) + +static uint64_t tens[] = { 10000000000000000000U, + 1000000000000000000U, + 100000000000000000U, + 10000000000000000U, + 1000000000000000U, + 100000000000000U, + 10000000000000U, + 1000000000000U, + 100000000000U, + 10000000000U, + 1000000000U, + 100000000U, + 10000000U, + 1000000U, + 100000U, + 10000U, + 1000U, + 100U, + 10U, + 1U }; + +static inline uint64_t get_dbits(double d) { + union + { + double dbl; + uint64_t i; + } dbl_bits = { d }; + + return dbl_bits.i; +} + +static Fp build_fp(double d) { + uint64_t bits = get_dbits(d); + + Fp fp; + fp.frac = bits & fracmask; + fp.exp = (bits & expmask) >> 52; + + if (fp.exp) { + fp.frac += hiddenbit; + fp.exp -= expbias; + + } else { + fp.exp = -expbias + 1; + } + + return fp; +} + +static void normalize(Fp *fp) { + while ((fp->frac & hiddenbit) == 0) { + fp->frac <<= 1; + fp->exp--; + } + + int shift = 64 - 52 - 1; + fp->frac <<= shift; + fp->exp -= shift; +} + +static void get_normalized_boundaries(Fp *fp, Fp *lower, Fp *upper) { + upper->frac = (fp->frac << 1) + 1; + upper->exp = fp->exp - 1; + + while ((upper->frac & (hiddenbit << 1)) == 0) { + upper->frac <<= 1; + upper->exp--; + } + + int u_shift = 64 - 52 - 2; + + upper->frac <<= u_shift; + upper->exp = upper->exp - u_shift; + + int l_shift = fp->frac == hiddenbit ? 2 : 1; + + lower->frac = (fp->frac << l_shift) - 1; + lower->exp = fp->exp - l_shift; + + lower->frac <<= lower->exp - upper->exp; + lower->exp = upper->exp; +} + +static Fp multiply(Fp *a, Fp *b) { + const uint64_t lomask = 0x00000000FFFFFFFF; + + uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); + uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); + uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); + uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); + + uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); + /* round up */ + tmp += 1U << 31; + + Fp fp = { ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), a->exp + b->exp + 64 }; + + return fp; +} + +static void round_digit(char *digits, + int ndigits, + uint64_t delta, + uint64_t rem, + uint64_t kappa, + uint64_t frac) { + while (rem < frac && delta - rem >= kappa && + (rem + kappa < frac || frac - rem > rem + kappa - frac)) { + digits[ndigits - 1]--; + rem += kappa; + } +} + +static int generate_digits(Fp *fp, Fp *upper, Fp *lower, char *digits, int *K) { + uint64_t wfrac = upper->frac - fp->frac; + uint64_t delta = upper->frac - lower->frac; + + Fp one; + one.frac = 1ULL << -upper->exp; + one.exp = upper->exp; + + uint64_t part1 = upper->frac >> -one.exp; + uint64_t part2 = upper->frac & (one.frac - 1); + + int idx = 0, kappa = 10; + uint64_t *divp; + /* 1000000000 */ + for (divp = tens + 10; kappa > 0; divp++) { + uint64_t div = *divp; + unsigned digit = part1 / div; + + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part1 -= digit * div; + kappa--; + + uint64_t tmp = (part1 << -one.exp) + part2; + if (tmp <= delta) { + *K += kappa; + round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); + + return idx; + } + } + + /* 10 */ + uint64_t *unit = tens + 18; + + while (true) { + part2 *= 10; + delta *= 10; + kappa--; + + unsigned digit = part2 >> -one.exp; + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part2 &= one.frac - 1; + if (part2 < delta) { + *K += kappa; + round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); + + return idx; + } + + unit--; + } +} + +static int grisu2(double d, char *digits, int *K) { + Fp w = build_fp(d); + + Fp lower, upper; + get_normalized_boundaries(&w, &lower, &upper); + + normalize(&w); + + int k; + Fp cp = find_cachedpow10(upper.exp, &k); + + w = multiply(&w, &cp); + upper = multiply(&upper, &cp); + lower = multiply(&lower, &cp); + + lower.frac++; + upper.frac--; + + *K = -k; + + return generate_digits(&w, &upper, &lower, digits, K); +} + +static int emit_digits(char *digits, int ndigits, char *dest, int K, bool neg) { + int exp = absv(K + ndigits - 1); + + /* write plain integer */ + if (K >= 0 && (exp < (ndigits + 7))) { + memcpy(dest, digits, ndigits); + memset(dest + ndigits, '0', K); + + return ndigits + K; + } + + /* write decimal w/o scientific notation */ + if (K < 0 && (K > -7 || exp < 4)) { + int offset = ndigits - absv(K); + /* fp < 1.0 -> write leading zero */ + if (offset <= 0) { + offset = -offset; + dest[0] = '0'; + dest[1] = '.'; + memset(dest + 2, '0', offset); + memcpy(dest + offset + 2, digits, ndigits); + + return ndigits + 2 + offset; + + /* fp > 1.0 */ + } else { + memcpy(dest, digits, offset); + dest[offset] = '.'; + memcpy(dest + offset + 1, digits + offset, ndigits - offset); + + return ndigits + 1; + } + } + + /* write decimal w/ scientific notation */ + ndigits = minv(ndigits, 18 - neg); + + int idx = 0; + dest[idx++] = digits[0]; + + if (ndigits > 1) { + dest[idx++] = '.'; + memcpy(dest + idx, digits + 1, ndigits - 1); + idx += ndigits - 1; + } + + dest[idx++] = 'e'; + + char sign = K + ndigits - 1 < 0 ? '-' : '+'; + dest[idx++] = sign; + + int cent = 0; + + if (exp > 99) { + cent = exp / 100; + dest[idx++] = cent + '0'; + exp -= cent * 100; + } + if (exp > 9) { + int dec = exp / 10; + dest[idx++] = dec + '0'; + exp -= dec * 10; + + } else if (cent) { + dest[idx++] = '0'; + } + + dest[idx++] = exp % 10 + '0'; + + return idx; +} + +static int filter_special(double fp, char *dest) { + if (fp == 0.0) { + dest[0] = '0'; + return 1; + } + + uint64_t bits = get_dbits(fp); + + bool nan = (bits & expmask) == expmask; + + if (!nan) { + return 0; + } + + if (bits & fracmask) { + dest[0] = 'n'; + dest[1] = 'a'; + dest[2] = 'n'; + + } else { + dest[0] = 'i'; + dest[1] = 'n'; + dest[2] = 'f'; + } + + return 3; +} + +int fpconv_dtoa(double d, char dest[24]) { + char digits[18]; + + int str_len = 0; + bool neg = false; + + if (get_dbits(d) & signmask) { + dest[0] = '-'; + str_len++; + neg = true; + } + + int spec = filter_special(d, dest + str_len); + + if (spec) { + return str_len + spec; + } + + int K = 0; + int ndigits = grisu2(d, digits, &K); + + str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); + + return str_len; +} diff --git a/deps/fpconv/fpconv_dtoa.h b/deps/fpconv/fpconv_dtoa.h new file mode 100644 index 000000000..328ed83f0 --- /dev/null +++ b/deps/fpconv/fpconv_dtoa.h @@ -0,0 +1,45 @@ +/* fpconv_dtoa.h -- floating point conversion utilities. + * + * Fast and accurate double to string conversion based on Florian Loitsch's + * Grisu-algorithm[1]. + * + * [1] https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2013-2019, night-shift + * Copyright (c) 2009, Florian Loitsch < florian.loitsch at inria dot fr > + * All rights reserved. + * + * Boost Software License - Version 1.0 - August 17th, 2003 + * + * Permission is hereby granted, free of charge, to any person or organization + * obtaining a copy of the software and accompanying documentation covered by + * this license (the "Software") to use, reproduce, display, distribute, + * execute, and transmit the Software, and to prepare derivative works of the + * Software, and to permit third-parties to whom the Software is furnished to + * do so, all subject to the following: + * + * The copyright notices in the Software and this entire statement, including + * the above license grant, this restriction and the following disclaimer, + * must be included in all copies of the Software, in whole or in part, and + * all derivative works of the Software, unless such copies or derivative + * works are solely in the form of machine-executable object code generated by + * a source language processor. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT + * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE + * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FPCONV_DTOA_H +#define FPCONV_DTOA_H + +int fpconv_dtoa(double fp, char dest[24]); + +#endif + +/* [1] http://florian.loitsch.com/publications/dtoa-pldi2010.pdf */ diff --git a/deps/fpconv/fpconv_powers.h b/deps/fpconv/fpconv_powers.h new file mode 100644 index 000000000..bc488f6dd --- /dev/null +++ b/deps/fpconv/fpconv_powers.h @@ -0,0 +1,133 @@ +/* fpconv_powers.h -- floating point conversion utilities. + * + * Fast and accurate double to string conversion based on Florian Loitsch's + * Grisu-algorithm[1]. + * + * [1] https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2021, Redis Labs + * Copyright (c) 2013-2019, night-shift + * Copyright (c) 2009, Florian Loitsch < florian.loitsch at inria dot fr > + * All rights reserved. + * + * Boost Software License - Version 1.0 - August 17th, 2003 + * + * Permission is hereby granted, free of charge, to any person or organization + * obtaining a copy of the software and accompanying documentation covered by + * this license (the "Software") to use, reproduce, display, distribute, + * execute, and transmit the Software, and to prepare derivative works of the + * Software, and to permit third-parties to whom the Software is furnished to + * do so, all subject to the following: + * + * The copyright notices in the Software and this entire statement, including + * the above license grant, this restriction and the following disclaimer, + * must be included in all copies of the Software, in whole or in part, and + * all derivative works of the Software, unless such copies or derivative + * works are solely in the form of machine-executable object code generated by + * a source language processor. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT + * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE + * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#define npowers 87 +#define steppowers 8 +#define firstpower -348 /* 10 ^ -348 */ + +#define expmax -32 +#define expmin -60 + + +typedef struct Fp { + uint64_t frac; + int exp; +} Fp; + +static Fp powers_ten[] = { + { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, + { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 }, + { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, + { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, + { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 }, + { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, + { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, + { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 }, + { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, + { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, + { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 }, + { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, + { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, + { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 }, + { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, + { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, + { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 }, + { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, + { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, + { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 }, + { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, + { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, + { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 }, + { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, + { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, + { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 }, + { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, + { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, + { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 }, + { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, + { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, + { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 }, + { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, + { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, + { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 }, + { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, + { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, + { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 }, + { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, + { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, + { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 }, + { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, + { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, + { 12648080533535911531U, 1066 } +}; + +/** + * Grisu needs a cache of precomputed powers-of-ten. + * This function, given an exponent and an integer k + * return the normalized floating-point approximation of the power of 10. + * @param exp + * @param k + * @return + */ +static Fp find_cachedpow10(int exp, int* k) +{ + const double one_log_ten = 0.30102999566398114; + + const int approx = -(exp + npowers) * one_log_ten; + int idx = (approx - firstpower) / steppowers; + + while(1) { + int current = exp + powers_ten[idx].exp + 64; + + if(current < expmin) { + idx++; + continue; + } + + if(current > expmax) { + idx--; + continue; + } + + *k = (firstpower + idx * steppowers); + + return powers_ten[idx]; + } +} diff --git a/src/Makefile b/src/Makefile index 2ffddef1b..7e17f1f83 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ ifeq ($(OPTIMIZATION),-O3) REDIS_CFLAGS+=-flto REDIS_LDFLAGS+=-flto endif -DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram +DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv NODEPS:=clean distclean # Default settings @@ -229,7 +229,7 @@ ifdef OPENSSL_PREFIX endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv # Determine systemd support and/or build preference (defaulting to auto-detection) BUILD_WITH_SYSTEMD=no @@ -395,7 +395,7 @@ endif # redis-server $(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) # redis-sentinel $(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME) diff --git a/src/debug.c b/src/debug.c index 9b31a3377..b642e4eac 100644 --- a/src/debug.c +++ b/src/debug.c @@ -34,6 +34,7 @@ #include "crc64.h" #include "bio.h" #include "quicklist.h" +#include "fpconv_dtoa.h" #include #include @@ -188,8 +189,8 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) ll2string(buf,sizeof(buf),vll); mixDigest(eledigest,buf,strlen(buf)); } - - snprintf(buf,sizeof(buf),"%.17g",score); + const int len = fpconv_dtoa(score, buf); + buf[len] = '\0'; mixDigest(eledigest,buf,strlen(buf)); xorDigest(digest,eledigest,20); zzlNext(zl,&eptr,&sptr); @@ -202,8 +203,8 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) while((de = dictNext(di)) != NULL) { sds sdsele = dictGetKey(de); double *score = dictGetVal(de); - - snprintf(buf,sizeof(buf),"%.17g",*score); + const int len = fpconv_dtoa(*score, buf); + buf[len] = '\0'; memset(eledigest,0,20); mixDigest(eledigest,sdsele,sdslen(sdsele)); mixDigest(eledigest,buf,strlen(buf)); diff --git a/src/networking.c b/src/networking.c index 3cdbe16ce..180782584 100644 --- a/src/networking.c +++ b/src/networking.c @@ -31,6 +31,7 @@ #include "atomicvar.h" #include "cluster.h" #include "script.h" +#include "fpconv_dtoa.h" #include #include #include @@ -855,7 +856,7 @@ void addReplyDouble(client *c, double d) { * but still avoid an extra memcpy of the whole number, we reserve space * for maximum header `$0000\r\n`, print double, add the resp header in * front of it, and then send the buffer with the right `start` offset. */ - int dlen = snprintf(dbuf+7,sizeof(dbuf) - 7,"%.17g",d); + dlen = fpconv_dtoa(d, dbuf+7); int digits = digits10(dlen); int start = 4 - digits; dbuf[start] = '$'; @@ -870,10 +871,15 @@ void addReplyDouble(client *c, double d) { dbuf[6] = '\n'; dbuf[dlen+7] = '\r'; dbuf[dlen+8] = '\n'; + dbuf[dlen+9] = '\0'; addReplyProto(c,dbuf+start,dlen+9-start); } else { - dlen = snprintf(dbuf,sizeof(dbuf),",%.17g\r\n",d); - addReplyProto(c,dbuf,dlen); + dbuf[0] = ','; + dlen = fpconv_dtoa(d, dbuf+1); + dbuf[dlen+1] = '\r'; + dbuf[dlen+2] = '\n'; + dbuf[dlen+3] = '\0'; + addReplyProto(c,dbuf,dlen+3); } } } diff --git a/src/rdb.c b/src/rdb.c index 77499c39e..24628a89b 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -31,6 +31,7 @@ #include "lzf.h" /* LZF compression library */ #include "zipmap.h" #include "endianconv.h" +#include "fpconv_dtoa.h" #include "stream.h" #include "functions.h" @@ -590,8 +591,10 @@ int rdbSaveDoubleValue(rio *rdb, double val) { /* Integer printing function is much faster, check if we can safely use it. */ if (double2ll(val, &lvalue)) ll2string((char*)buf+1,sizeof(buf)-1,lvalue); - else - snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); + else { + const int dlen = fpconv_dtoa(val, (char*)buf+1); + buf[dlen+1] = '\0'; + } buf[0] = strlen((char*)buf+1); len = buf[0]+1; } diff --git a/src/rio.c b/src/rio.c index bcda3767b..de4713fec 100644 --- a/src/rio.c +++ b/src/rio.c @@ -46,6 +46,7 @@ #include "fmacros.h" +#include "fpconv_dtoa.h" #include #include #include @@ -493,7 +494,7 @@ size_t rioWriteBulkLongLong(rio *r, long long l) { size_t rioWriteBulkDouble(rio *r, double d) { char dbuf[128]; unsigned int dlen; - - dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); + dlen = fpconv_dtoa(d, dbuf); + dbuf[dlen] = '\0'; return rioWriteBulkString(r,dbuf,dlen); } diff --git a/src/script_lua.c b/src/script_lua.c index 67209615b..70c7185f9 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -28,6 +28,7 @@ */ #include "script_lua.h" +#include "fpconv_dtoa.h" #include "server.h" #include "sha1.h" @@ -804,8 +805,8 @@ static robj **luaArgsToRedisArgv(lua_State *lua, int *argc) { /* We can't use lua_tolstring() for number -> string conversion * since Lua uses a format specifier that loses precision. */ lua_Number num = lua_tonumber(lua,j+1); - - obj_len = snprintf(dbuf,sizeof(dbuf),"%.17g",(double)num); + obj_len = fpconv_dtoa((double)num, dbuf); + dbuf[obj_len] = '\0'; obj_s = dbuf; } else { obj_s = (char*)lua_tolstring(lua,j+1,&obj_len); diff --git a/src/util.c b/src/util.c index 092814251..ca2a8366b 100644 --- a/src/util.c +++ b/src/util.c @@ -28,6 +28,7 @@ */ #include "fmacros.h" +#include "fpconv_dtoa.h" #include #include #include @@ -617,8 +618,10 @@ int d2string(char *buf, size_t len, double value) { /* Integer printing function is much faster, check if we can safely use it. */ if (double2ll(value, &lvalue)) len = ll2string(buf,len,lvalue); - else - len = snprintf(buf,len,"%.17g",value); + else { + len = fpconv_dtoa(value, buf); + buf[len] = '\0'; + } } return len; diff --git a/src/util.h b/src/util.h index e009354e8..4ff8a88e6 100644 --- a/src/util.h +++ b/src/util.h @@ -43,7 +43,7 @@ * This should be the size of the buffer for sprintf with %f */ #define MAX_DOUBLE_CHARS 400 -/* The maximum number of characters needed to for d2string call. +/* The maximum number of characters needed to for d2string/fpconv_dtoa call. * Since it uses %g and not %f, some 40 chars should be enough. */ #define MAX_D2STRING_CHARS 128 diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 104d372e1..1dd6bb1d7 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -29,7 +29,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rd "0","set_zipped_2","set","100000","200000","300000","400000", "0","set_zipped_3","set","1000000000","2000000000","3000000000","4000000000","5000000000","6000000000", "0","string","string","Hello World" -"0","zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5000000000", +"0","zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5e+9", "0","zset_zipped","zset","a","1","b","2","c","3", } } diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 0e2457516..66de682db 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1516,7 +1516,11 @@ start_server {tags {"zset"}} { assert_encoding $encoding zscoretest for {set i 0} {$i < $elements} {incr i} { - assert_equal [lindex $aux $i] [r zscore zscoretest $i] + # If an IEEE 754 double-precision number is converted to a decimal string with at + # least 17 significant digits (reply of zscore), and then converted back to double-precision representation, + # the final result replied via zscore command must match the original number present on the $aux list. + # Given Tcl is mostly very relaxed about types (everything is a string) we need to use expr to convert a string to float. + assert_equal [expr [lindex $aux $i]] [expr [r zscore zscoretest $i]] } } @@ -1531,7 +1535,8 @@ start_server {tags {"zset"}} { assert_encoding $encoding zscoretest for {set i 0} {$i < $elements} {incr i} { - assert_equal [lindex $aux $i] [r zmscore zscoretest $i] + # Check above notes on IEEE 754 double-precision comparison + assert_equal [expr [lindex $aux $i]] [expr [r zscore zscoretest $i]] } } @@ -1547,7 +1552,8 @@ start_server {tags {"zset"}} { r debug reload assert_encoding $encoding zscoretest for {set i 0} {$i < $elements} {incr i} { - assert_equal [lindex $aux $i] [r zscore zscoretest $i] + # Check above notes on IEEE 754 double-precision comparison + assert_equal [expr [lindex $aux $i]] [expr [r zscore zscoretest $i]] } } {} {needs:debug}