# [Git][NTPsec/ntpsec][master] 3 commits: remove unused, expensive and un-needed percentil function.

Gary E. Miller gitlab at mg.gitlab.com
Fri Sep 9 03:23:25 UTC 2016

```Gary E. Miller pushed to branch master at NTPsec / ntpsec

Commits:
1826df4c by Gary E. Miller at 2016-09-08T19:16:16-07:00
remove unused, expensive and un-needed percentil function.

- - - - -
cdba7769 by Gary E. Miller at 2016-09-08T20:07:08-07:00
Make percentiles() return a dictionary instead of a tuple.

This simplifies the notation, but stuip %-substition does not work
with dictionaries.

namedtuples have a baffling restriction of 4 elements, so unuseable.

- - - - -
13939f76 by Gary E. Miller at 2016-09-08T20:22:06-07:00
Do multiplier in a loop, not inline.

Next up is likely putting all the autoscale/percentages in a
single function

- - - - -

2 changed files:

- ntpstats/ntpviz
- pylib/statfiles.py

Changes:

=====================================
ntpstats/ntpviz
=====================================
--- a/ntpstats/ntpviz
+++ b/ntpstats/ntpviz
@@ -120,10 +120,9 @@ set rmargin 12
values = [float(line.split()[1]) for line in self.loopstats]
values.sort()

-        ninetynine, one = \
-                    self.percentiles( (99, 1), values)
+        percs = self.percentiles( (99, 1), values)

-        if 1e-6 > ninetynine and -1e-6 < one:
+        if 1e-6 > percs[99] and -1e-6 < percs[1]:
# go to nanosec
unit = "ns"
multiplier = 1e9
@@ -133,10 +132,9 @@ set rmargin 12
values_f = [float(line.split()[2]) for line in self.loopstats]
values_f.sort()

-        ninetynine_f, one_f = \
-                    self.percentiles( (99, 1), values_f)
+        percs_f = self.percentiles( (99, 1), values_f)

-        if 1 > ninetynine_f and -1 < one_f:
+        if 1 > percs_f[99] and -1 < percs_f[1]:
# go to nanosec
unit_f = "ppb"
multiplier_f = 1e3
@@ -204,22 +202,22 @@ plot \\
unit = "ppm"
rnd = 3

-        ninetynine, ninetyfive, five, one = \
-                    self.percentiles( (99, 95, 5, 1), values)
+        percs = self.percentiles( (99, 95, 5, 1), values)

-        if 1 > ninetynine and -1 < one:
+        if 1 > percs[99] and -1 < percs[1]:
# go to ppb
multiplier = 1e3
unit = "ppb"
rnd = 0

-        ninetynine  = round( ninetynine * multiplier, rnd)
-        ninetyfive  = round( ninetyfive * multiplier, rnd)
-        five        = round(       five * multiplier, rnd)
-        one         = round(        one * multiplier, rnd)
+        percs.update({k: round( v * multiplier, rnd) for k, v in percs.items()})
+        ninetynine  = percs[99]
+        ninetyfive  = percs[95]
+        five        = percs[5]
+        one         = percs[1]

-        nf_m_f     = ninetyfive - five
-        nn_m_o     = ninetynine - one
+        nf_m_f     = percs[95] - percs[5]
+        nn_m_o     = percs[99] - percs[1]
plot_template = NTPViz.Common + """\
set title "%(sitename)s: Local Clock Frequency Offset"
set ytics format "%%1.1f %(unit)s" nomirror
@@ -258,37 +256,36 @@ plot \
values = [float(line.split()[fld - 1]) for line in self.loopstats]
values.sort()

-        ninetynine, ninetyfive, five, one = \
-                    self.percentiles( (99, 95, 5, 1), values)
+        percs = self.percentiles( (99, 95, 5, 1), values)

-        if 1e-3 <= ninetynine or -1e-3 >= one:
+        if 1e-3 <= percs[99] or -1e-3 >= percs[1]:
# go to millisec
multiplier = 1e3
if freq:
unit = "ppt"
else:
unit = "ms"
-        else:
-            if 1e-6 > ninetynine and -1e-6 < one:
-                # go to nanosec, or ppb
-                multiplier = 1e9
-                rnd = 0
-                if freq:
-                    unit = "ppb"
-                else:
-                    unit = "ns"
+        elif 1e-6 > percs[99] and -1e-6 < percs[1]:
+            # go to nanosec, or ppb
+            multiplier = 1e9
+            rnd = 0
+            if freq:
+                unit = "ppb"
+            else:
+                unit = "ns"

-        ninetynine  = round( ninetynine * multiplier, rnd)
-        ninetyfive  = round( ninetyfive * multiplier, rnd)
-        five        = round(       five * multiplier, rnd)
-        one         = round(        one * multiplier, rnd)
+        percs.update({k: round( v * multiplier, rnd) for k, v in percs.items()})
+        ninetynine  = percs[99]
+        ninetyfive  = percs[95]
+        five        = percs[5]
+        one         = percs[1]

nf_m_f     = ninetyfive - five
nn_m_o     = ninetynine - one

plot_template = NTPViz.Common + """\
set title "%(sitename)s: %(title)s"
-set ytics format "%%1.0f %(unit)s" nomirror
+set ytics format "%%1.1f %(unit)s" nomirror
set key top right box
set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
set style line 2 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
@@ -345,7 +342,7 @@ plot \
plot_data = plot_data[:-2]

unit = "μs"
-        multiplier = 1000000
+        multiplier = 1e6
rtt = 0
percentages = ""
if len(peerlist) == 1:
@@ -355,32 +352,27 @@ plot \
values = [float(line.split()[fld - 1]) for line in peerdict[ip]]
values.sort()

-            ninetynine, ninetyfive, five, one = \
-                        self.percentiles( (99,95, 5, 1), values)
-            ninetynine  = round( ninetynine * multiplier, 3)
-            ninetyfive  = round( ninetyfive * multiplier, 3)
-            five        = round(       five * multiplier, 3)
-            one         = round(        one * multiplier, 3)
+            percs = self.percentiles( (99, 95, 50, 5, 1), values)

-            if 1000 <= ninetynine or -1000 >= one:
+            if 1e-3 <= percs[99] or -1e-3 >= percs[1]:
# go to millisec
unit = "ms"
-                multiplier = 1000
-            else:
-                if 1 > ninetynine and -1 < one:
-                    # go to nanosec
-                    unit = "ns"
-                    multiplier = 1e9
+                multiplier = 1e3
+            elif 1e-6 > percs[99] and -1e-6 < percs[1]:
+                # go to nanosec
+                unit = "ns"
+                multiplier = 1e9
+
+            rnd = 3
+            percs.update({k: round(v*multiplier, rnd) for k, v in percs.items()})
+            ninetynine  = percs[99]
+            ninetyfive  = percs[95]
+            fifty       = percs[50]
+            five        = percs[5]
+            one         = percs[1]

if "offset" == type:
# fld == 4
-                ninetynine, ninetyfive, fifty, five, one = \
-                            self.percentiles( (99,95, 50, 5, 1), values)
-                ninetynine  = round( ninetynine * multiplier, 3)
-                ninetyfive  = round( ninetyfive * multiplier, 3)
-                fifty       = round(      fifty * multiplier, 3)
-                five        = round(       five * multiplier, 3)
-                one         = round(        one * multiplier, 3)

nf_m_f     = ninetyfive - five
nn_m_o     = ninetynine - one
@@ -448,15 +440,14 @@ plot \
values = [float(line.split()[1]) for line in self.loopstats]
values.sort()

-        ninetynine, ninetyfive, five, one = \
-                    self.percentiles( (99,95, 5, 1), values)
+        percs = self.percentiles( (99, 95, 5, 1), values)

unit = "μs"
multiplier = 1e6
rnd = 2
rnd1 = 7        # round to 100 ns boxes
boxwidth = 1e-7
-        if 1e-6 > ninetynine and -1e-6 < one:
+        if 1e-6 > percs[99] and -1e-6 < percs[1]:
# go to nanosec
unit = "ns"
multiplier = 1e9
@@ -464,10 +455,11 @@ plot \
rnd1 = 9        # round to 1 ns boxes
boxwidth = 1e-9

-        ninetynine  = round( ninetynine * multiplier, rnd)
-        ninetyfive  = round( ninetyfive * multiplier, rnd)
-        five        = round(       five * multiplier, rnd)
-        one         = round(        one * multiplier, rnd)
+        percs.update({k: round(v*multiplier, rnd) for k, v in percs.items()})
+        ninetynine  = percs[99]
+        ninetyfive  = percs[95]
+        five        = percs[5]
+        one         = percs[1]

cnt = collections.Counter()
for value in values:

=====================================
pylib/statfiles.py
=====================================
--- a/pylib/statfiles.py
+++ b/pylib/statfiles.py
@@ -7,7 +7,7 @@ SPDX-License-Identifier: BSD-2-Clause
"""
from __future__ import print_function, division

-import os, sys, time, glob, calendar, subprocess, socket, gzip, datetime
+import calendar, datetime, glob, gzip, os, socket, subprocess, sys, time

class NTPStats:
"Gather statistics for a specified NTP site"
@@ -123,22 +123,14 @@ class NTPStats:
if ts < m:
m = ts
return m
-    def percentile(self, n, percentile, entries):
-        "Return given percentiles of a given row in a given set of entries."
-        "If you call this twice on the same data set you should use"
-        # Row is decremented so we match GNUPLOT's 1-origin indexing.
-        values = [float(line.split()[n-1]) for line in entries]
-        values.sort()
-        return values[int(len(values) * (percentile/100))]
def percentiles(self, percents, values):
"Return given percentiles of a given row in a given set of entries."
"assuming values are already split and sorted"
-        ret = []
+        ret = {}
length = len(values)
for perc in percents:
-            ret.append( values[int(length * (perc/100))] )
-        return tuple( ret )
+            ret[perc] = values[int(length * (perc/100))]
+        return ret
def peersplit(self):
"Return a dictionary mapping peerstats IPs to entry subsets."
peermap = {}

View it on GitLab: https://gitlab.com/NTPsec/ntpsec/compare/59dc7e803b2c2c1aaf68d395fbb7649a9ea79f9c...13939f7672e460913e20d4e0427c3cd98a400c83
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.ntpsec.org/pipermail/vc/attachments/20160909/e2cc9e6e/attachment.html>
```