[Git][NTPsec/ntpsec][master] Finished tests for statfiles.py (for real this time)
Ian Bruene
gitlab at mg.gitlab.com
Tue Sep 12 19:22:15 UTC 2017
Ian Bruene pushed to branch master at NTPsec / ntpsec
Commits:
774f957f by Ian Bruene at 2017-09-12T14:20:43-05:00
Finished tests for statfiles.py (for real this time)
- - - - -
3 changed files:
- pylib/statfiles.py
- tests/pylib/jigs.py
- tests/pylib/test_statfiles.py
Changes:
=====================================
pylib/statfiles.py
=====================================
--- a/pylib/statfiles.py
+++ b/pylib/statfiles.py
@@ -59,6 +59,53 @@ class NTPStats:
"get Unix time from converted line."
return float(line.split()[0])
+ @staticmethod
+ def percentiles(percents, values):
+ "Return given percentiles of a given row in a given set of entries."
+ "assuming values are already split and sorted"
+ ret = {}
+ length = len(values)
+ if 1 >= length:
+ # uh, oh...
+ if 1 == length:
+ # just one data value, set all to that one value
+ v = values[0]
+ else:
+ # no data, set all to zero
+ v = 0
+ for perc in percents:
+ ret["p" + str(perc)] = v
+ else:
+ for perc in percents:
+ if perc == 100:
+ ret["p100"] = values[length - 1]
+ else:
+ ret["p" + str(perc)] = values[int(length * (perc/100))]
+ return ret
+
+ @staticmethod
+ def ip_label(key):
+ "Produce appropriate label for an IP address."
+ # If it's a new-style NTPsep clock label, pass it through,
+ # Otherwise we expect it to be an IP address and the next guard fires
+ if key[0].isdigit():
+ # TO BE REMOVED SOMEDAY
+ # Clock address - only possible if we're looking at a logfile made
+ # by NTP Classic or an NTPsec version configured with
+ # --enable-classic-mode. Nasty that we have to emit a numeric
+ # driver type here.
+ if key.startswith("127.127."):
+ (_, _, t, u) = key.split(".")
+ return "REFCLOCK(type=%s,unit=%s)" % (t, u)
+ # Ordinary IP address - replace with primary hostname.
+ # Punt if the lookup fails.
+ try:
+ (hostname, _, _) = socket.gethostbyaddr(key)
+ return hostname
+ except socket.herror:
+ pass
+ return key # Someday, be smarter than this.
+
def __init__(self, statsdir, sitename=None,
period=None, starttime=None, endtime=None):
"Grab content of logfiles, sorted by timestamp."
@@ -104,6 +151,7 @@ class NTPStats:
lines += open(logpart, 'r').readlines()
except IOError:
sys.stderr.write("ntpviz: WARNING: could not read %s\n"
+
% logpart)
pass
@@ -138,29 +186,6 @@ class NTPStats:
lines1.sort()
setattr(self, stem, lines1)
- def percentiles(self, percents, values):
- "Return given percentiles of a given row in a given set of entries."
- "assuming values are already split and sorted"
- ret = {}
- length = len(values)
- if 1 >= length:
- # uh, oh...
- if 1 == length:
- # just one data value, set all to that one value
- v = values[0]
- else:
- # no data, set all to zero
- v = 0
- for perc in percents:
- ret["p" + str(perc)] = v
- else:
- for perc in percents:
- if perc == 100:
- ret["p100"] = values[length - 1]
- else:
- ret["p" + str(perc)] = values[int(length * (perc/100))]
- return ret
-
def peersplit(self):
"Return a dictionary mapping peerstats IPs to entry subsets."
"This is very expensive, so cache the result"
@@ -206,28 +231,6 @@ class NTPStats:
pass
return tempsmap
- def ip_label(self, key):
- "Produce appropriate label for an IP address."
- # If it's a new-style NTPsep clock label, pass it through,
- # Otherwise we expect it to be an IP address and the next guard fires
- if key[0].isdigit():
- # TO BE REMOVED SOMEDAY
- # Clock address - only possible if we're looking at a logfile made
- # by NTP Classic or an NTPsec version configured with
- # --enable-classic-mode. Nasty that we have to emit a numeric
- # driver type here.
- if key.startswith("127.127."):
- (_, _, t, u) = key.split(".")
- return "REFCLOCK(type=%s,unit=%s)" % (t, u)
- # Ordinary IP address - replace with primary hostname.
- # Punt if the lookup fails.
- try:
- (hostname, _, _) = socket.gethostbyaddr(key)
- return hostname
- except socket.herror:
- pass
- return key # Someday, be smarter than this.
-
def iso_to_posix(s):
"Accept timestamps in ISO 8661 format or numeric POSIX time. UTC only."
=====================================
tests/pylib/jigs.py
=====================================
--- a/tests/pylib/jigs.py
+++ b/tests/pylib/jigs.py
@@ -96,6 +96,7 @@ class HasherJig:
class SocketModuleJig:
error = socket.error
gaierror = socket._socket.gaierror
+ herror = socket.herror
SOCK_DGRAM = socket.SOCK_DGRAM
IPPROTO_UDP = socket.IPPROTO_UDP
AF_UNSPEC = socket.AF_UNSPEC
@@ -119,6 +120,8 @@ class SocketModuleJig:
self.inet_ntop_calls = []
self.getfqdn_calls = []
self.getfqdn_returns = []
+ self.ghba_calls = []
+ self.ghba_returns = []
def getaddrinfo(self, host, port, family=None, socktype=None,
proto=None, flags=None):
@@ -160,6 +163,13 @@ class SocketModuleJig:
self.getfqdn_calls.append(name)
return self.getfqdn_returns.pop(0)
+ def gethostbyaddr(self, addr):
+ self.ghba_calls.append(addr)
+ ret = self.ghba_returns.pop(0)
+ if ret is None:
+ raise self.herror
+ return ret
+
class GetpassModuleJig:
def __init__(self):
=====================================
tests/pylib/test_statfiles.py
=====================================
--- a/tests/pylib/test_statfiles.py
+++ b/tests/pylib/test_statfiles.py
@@ -33,6 +33,17 @@ class TestPylibStatfiles(unittest.TestCase):
class TestNTPStats(unittest.TestCase):
target = ntp.statfiles.NTPStats
+ open_calls = []
+ open_returns = []
+
+ def open_jig(self, filename, filemode):
+ self.open_calls.append((filename, filemode))
+ ret = self.open_returns.pop(0)
+ if ret is None:
+ raise IOError
+ else:
+ return ret
+
def test_unixize(self):
f = self.target.unixize
@@ -58,18 +69,45 @@ class TestNTPStats(unittest.TestCase):
# Test
self.assertEqual(f("12345.6789 blah blah"), 12345.6789)
- def test___init__(self):
- open_calls = []
- open_returns = []
+ def test_percentiles(self):
+ f = self.target.percentiles
+
+ # Test empty
+ self.assertEqual(f([], []), {})
+ # Test 1 item, empty percentile
+ self.assertEqual(f([], [42]), {})
+ # Test 1 item, non-empty percentile
+ self.assertEqual(f([10, 90], [42]), {"p10": 42, "p90": 42})
+ # Test several items, empty percentile
+ self.assertEqual(f([], [1, 23, 42, 99]), {})
+ # Test several items, non-empty percentile
+ self.assertEqual(f([10, 25, 50, 90, 100], [1, 23, 42, 99]),
+ {"p10": 1, "p25": 23, "p90": 99,
+ "p50": 42, "p100": 99})
+
+ def test_ip_label(self):
+ f = self.target.ip_label
- def open_jig(filename, filemode):
- open_calls.append((filename, filemode))
- ret = open_returns.pop(0)
- if ret is None:
- raise IOError
- else:
- return ret
+ fakesockmod = jigs.SocketModuleJig()
+ try:
+ socktemp = ntp.statfiles.socket
+ ntp.statfiles.socket = fakesockmod
+ # Test no label
+ self.assertEqual(f("blah"), "blah")
+ # Test hostname, success
+ fakesockmod.ghba_returns = [("result.com", None, None)]
+ self.assertEqual(f("1.2.3.4"), "result.com")
+ # Test hostname, failure
+ fakesockmod.ghba_returns = [None]
+ self.assertEqual(f("1.2.3.4"), "1.2.3.4")
+ # Test old style NTP
+ fakesockmod.ghba_returns = [("foo.org", None, None)]
+ self.assertEqual(f("127.127.42.23"), "REFCLOCK(type=42,unit=23)")
+ finally:
+ ntp.statfiles.socket = socktemp
+ def test___init__(self):
+ # Create jigs
fakegzipmod = jigs.GzipModuleJig()
fakesockmod = jigs.SocketModuleJig()
logjig = jigs.FileJig()
@@ -91,7 +129,7 @@ class TestNTPStats(unittest.TestCase):
globtemp = ntp.statfiles.glob
ntp.statfiles.glob = fakeglobmod
opentemp = open
- ntp.statfiles.open = open_jig
+ ntp.statfiles.open = self.open_jig
# Test simplest
TDP = self.target.DefaultPeriod
faketimemod.time_returns = [TDP * 2]
@@ -99,21 +137,25 @@ class TestNTPStats(unittest.TestCase):
fakeosmod.path.getmtime_returns = [TDP+1, TDP+2, TDP+3, TDP+4,
TDP+5, TDP+6, TDP+7, TDP+8,
TDP+9, TDP+10, TDP+11, TDP-1]
- open_returns = [jigs.FileJig(["40594 10\n", "40594 11"]),
- jigs.FileJig(["40594 30\n", "40594 31"]),
- jigs.FileJig(["40594 40\n", "40594 41"]),
- jigs.FileJig(["40594 50\n", "40594 51"]),
- jigs.FileJig(["40594 60\n", "40594 61"]),
- jigs.FileJig(["40594 70\n", "40594 71"]),
- jigs.FileJig(["40594 80\n", "40594 81"]),
- jigs.FileJig(["604801.25 40594 90\n", "#blah",
- "604802.25 40594 91"]),
- jigs.FileJig(["604803.25 40594 100\n", "#blah",
- "604804.25 40594 101"]),
- jigs.FileJig(["604805.25 40594 110\n", "#blah",
- "604806.25 40594 111"]),
- jigs.FileJig(["604807.25 40594 120\n", "#blah",
- "604808.25 40594 121"])]
+ self.open_returns = [jigs.FileJig(["40594 10\n", "40594 11"]),
+ jigs.FileJig(["40594 30\n", "40594 31"]),
+ jigs.FileJig(["40594 40\n", "40594 41"]),
+ jigs.FileJig(["40594 50\n", "40594 51"]),
+ jigs.FileJig(["40594 60\n", "40594 61"]),
+ jigs.FileJig(["40594 70\n", "40594 71"]),
+ jigs.FileJig(["40594 80\n", "40594 81"]),
+ jigs.FileJig(["604801.25 40594 90\n",
+ "#blah",
+ "604802.25 40594 91"]),
+ jigs.FileJig(["604803.25 40594 100\n",
+ "#blah",
+ "604804.25 40594 101"]),
+ jigs.FileJig(["604805.25 40594 110\n",
+ "#blah",
+ "604806.25 40594 111"]),
+ jigs.FileJig(["604807.25 40594 120\n",
+ "#blah",
+ "604808.25 40594 121"])]
fakegzipmod.files_returned = [jigs.FileJig(["40594 20\n",
"40594 21"])]
fakeglobmod.glob_returns = [("/foo/bar/clockstats.0",
@@ -166,7 +208,7 @@ class TestNTPStats(unittest.TestCase):
faketimemod.time_returns = [TDP * 2]
fakesockmod.fqdn_returns = ["jabber"]
fakeosmod.path.isdir_returns = [True]
- open_returns = [None]
+ self.open_returns = [None]
fakeglobmod.glob_returns = [(["/foo/bar/clockstats.0"]),
([]), ([]), ([]), ([]), ([])]
fakeosmod.path.getmtime_returns = [101, 102, 103, 104, 105, 106]
@@ -213,6 +255,265 @@ class TestNTPStats(unittest.TestCase):
ntp.statfiles.open = opentemp
sys.stderr = errtemp
+ def test_peersplit(self):
+ # Yes, this copies much of test___init__. I already know it works
+ # and __init__ is very complicated
+ # Create jigs
+ fakegzipmod = jigs.GzipModuleJig()
+ fakesockmod = jigs.SocketModuleJig()
+ logjig = jigs.FileJig()
+ faketimemod = jigs.TimeModuleJig()
+ fakeosmod = jigs.OSModuleJig()
+ fakeglobmod = jigs.GlobModuleJig()
+ try:
+ # Splice in jigs
+ gziptemp = ntp.statfiles.gzip
+ ntp.statfiles.gzip = fakegzipmod
+ socktemp = ntp.statfiles.socket
+ ntp.statfiles.socket = fakesockmod
+ errtemp = sys.stderr
+ sys.stderr = logjig
+ timetemp = ntp.statfiles.time
+ ntp.statfiles.time = faketimemod
+ ostemp = ntp.statfiles.os
+ ntp.statfiles.os = fakeosmod
+ globtemp = ntp.statfiles.glob
+ ntp.statfiles.glob = fakeglobmod
+ opentemp = open
+ ntp.statfiles.open = self.open_jig
+ # Test simplest
+ TDP = self.target.DefaultPeriod
+ faketimemod.time_returns = [TDP * 2]
+ fakeosmod.path.isdir_returns = [True]
+ fakeosmod.path.getmtime_returns = [TDP+1, TDP+2, TDP+3, TDP+4,
+ TDP+5, TDP+6, TDP+7, TDP+8,
+ TDP+9, TDP+10, TDP+11, TDP-1]
+ self.open_returns = [jigs.FileJig(["40594 10\n", "40594 11"]),
+ jigs.FileJig(["40594 30 1.2.3.4\n",
+ "50594 -12\n",
+ "40594 31 1.2.3.4\n",
+ "40594 31.5 5.6.7.8\n"]),
+ jigs.FileJig(["40594 40\n", "40594 41"]),
+ jigs.FileJig(["40594 50\n", "40594 51"]),
+ jigs.FileJig(["40594 60\n", "40594 61"]),
+ jigs.FileJig(["40594 70\n", "40594 71"]),
+ jigs.FileJig(["40594 80\n", "40594 81"]),
+ jigs.FileJig(["604801.25 40594 90\n",
+ "#blah",
+ "604802.25 40594 91"]),
+ jigs.FileJig(["604803.25 40594 100\n",
+ "#blah",
+ "604804.25 40594 101"]),
+ jigs.FileJig(["604805.25 40594 110\n",
+ "#blah",
+ "604806.25 40594 111"]),
+ jigs.FileJig(["604807.25 40594 120\n",
+ "#blah",
+ "604808.25 40594 121"])]
+ fakegzipmod.files_returned = [jigs.FileJig(["40594 20\n",
+ "40594 21"])]
+ fakeglobmod.glob_returns = [("/foo/bar/clockstats.0",
+ "/foo/bar/clockstats.1gz"),
+ ("/foo/bar/peerstats.0",
+ "/foo/bar/peerstats.1"),
+ ("/foo/bar/loopstats.0",
+ "/foo/bar/loopstats.1"),
+ ("/foo/bar/rawstats.0",
+ "/foo/bar/rawstats.1"),
+ ("/foo/bar/temps0",
+ "/foo/bar/temps1"),
+ ("/foo/bar/gpsd0",
+ "/foo/bar/gpsd1")] # time kicked
+ cls = self.target("/foo/bar")
+ # Test
+ self.assertEqual(cls.peersplit(),
+ {"1.2.3.4": [[604830000, '604830.0', "1.2.3.4"],
+ [604831000, '604831.0', "1.2.3.4"]],
+ "5.6.7.8": [[604831500, '604831.5', "5.6.7.8"]]})
+ self.target.peermap["1.2.3.4"][0][0] = 42
+ # Test that it uses cache
+ self.assertEqual(cls.peersplit(),
+ {"1.2.3.4": [[42, '604830.0', "1.2.3.4"],
+ [604831000, '604831.0', "1.2.3.4"]],
+ "5.6.7.8": [[604831500, '604831.5', "5.6.7.8"]]})
+ finally:
+ ntp.statfiles.gzip = gziptemp
+ ntp.statfiles.socket = socktemp
+ ntp.statfiles.os = ostemp
+ ntp.statfiles.time = timetemp
+ ntp.statfiles.glob = fakeglobmod
+ ntp.statfiles.open = opentemp
+ sys.stderr = errtemp
+
+ def test_gpssplit(self):
+ # Yes, this copies much of test___init__. I already know it works
+ # and __init__ is very complicated
+ # Create jigs
+ fakegzipmod = jigs.GzipModuleJig()
+ fakesockmod = jigs.SocketModuleJig()
+ logjig = jigs.FileJig()
+ faketimemod = jigs.TimeModuleJig()
+ fakeosmod = jigs.OSModuleJig()
+ fakeglobmod = jigs.GlobModuleJig()
+ try:
+ # Splice in jigs
+ gziptemp = ntp.statfiles.gzip
+ ntp.statfiles.gzip = fakegzipmod
+ socktemp = ntp.statfiles.socket
+ ntp.statfiles.socket = fakesockmod
+ errtemp = sys.stderr
+ sys.stderr = logjig
+ timetemp = ntp.statfiles.time
+ ntp.statfiles.time = faketimemod
+ ostemp = ntp.statfiles.os
+ ntp.statfiles.os = fakeosmod
+ globtemp = ntp.statfiles.glob
+ ntp.statfiles.glob = fakeglobmod
+ opentemp = open
+ ntp.statfiles.open = self.open_jig
+ # Test simplest
+ TDP = self.target.DefaultPeriod
+ faketimemod.time_returns = [TDP * 2]
+ fakeosmod.path.isdir_returns = [True]
+ fakeosmod.path.getmtime_returns = [TDP+1, TDP+2, TDP+3, TDP+4,
+ TDP+5, TDP+6, TDP+7, TDP+8,
+ TDP+9, TDP+10, TDP+11, TDP-1]
+ self.open_returns = [jigs.FileJig(["40594 10\n", "40594 11"]),
+ jigs.FileJig(["40594 30\n", "40594 31"]),
+ jigs.FileJig(["40594 40\n", "40594 41"]),
+ jigs.FileJig(["40594 50\n", "40594 51"]),
+ jigs.FileJig(["40594 60\n", "40594 61"]),
+ jigs.FileJig(["40594 70\n", "40594 71"]),
+ jigs.FileJig(["40594 80\n", "40594 81"]),
+ jigs.FileJig(["604801.25 40594 90\n",
+ "#blah",
+ "604802.25 40594 91"]),
+ jigs.FileJig(["604803.25 40594 100\n",
+ "#blah",
+ "604804.25 40594 101"]),
+ jigs.FileJig(["604805.25 40594 110\n",
+ "#blah",
+ "604806.25 40594 111"]),
+ jigs.FileJig(["604807.25 40594 120\n",
+ "#blah",
+ "604808.25 40594 121"])]
+ fakegzipmod.files_returned = [jigs.FileJig(["40594 20\n",
+ "40594 21"])]
+ fakeglobmod.glob_returns = [("/foo/bar/clockstats.0",
+ "/foo/bar/clockstats.1gz"),
+ ("/foo/bar/peerstats.0",
+ "/foo/bar/peerstats.1"),
+ ("/foo/bar/loopstats.0",
+ "/foo/bar/loopstats.1"),
+ ("/foo/bar/rawstats.0",
+ "/foo/bar/rawstats.1"),
+ ("/foo/bar/temps0",
+ "/foo/bar/temps1"),
+ ("/foo/bar/gpsd0",
+ "/foo/bar/gpsd1")] # time kicked
+ cls = self.target("/foo/bar")
+ self.assertEqual(cls.gpssplit(),
+ {"40594": [[604805250, '604805.25', '40594',
+ '110'],
+ [604806250, '604806.25', '40594',
+ '111']]})
+ finally:
+ ntp.statfiles.gzip = gziptemp
+ ntp.statfiles.socket = socktemp
+ ntp.statfiles.os = ostemp
+ ntp.statfiles.time = timetemp
+ ntp.statfiles.glob = fakeglobmod
+ ntp.statfiles.open = opentemp
+ sys.stderr = errtemp
+
+ def test_tempssplit(self):
+ # Yes, this copies much of test___init__. I already know it works
+ # and __init__ is very complicated
+ # Create jigs
+ fakegzipmod = jigs.GzipModuleJig()
+ fakesockmod = jigs.SocketModuleJig()
+ logjig = jigs.FileJig()
+ faketimemod = jigs.TimeModuleJig()
+ fakeosmod = jigs.OSModuleJig()
+ fakeglobmod = jigs.GlobModuleJig()
+ try:
+ # Splice in jigs
+ gziptemp = ntp.statfiles.gzip
+ ntp.statfiles.gzip = fakegzipmod
+ socktemp = ntp.statfiles.socket
+ ntp.statfiles.socket = fakesockmod
+ errtemp = sys.stderr
+ sys.stderr = logjig
+ timetemp = ntp.statfiles.time
+ ntp.statfiles.time = faketimemod
+ ostemp = ntp.statfiles.os
+ ntp.statfiles.os = fakeosmod
+ globtemp = ntp.statfiles.glob
+ ntp.statfiles.glob = fakeglobmod
+ opentemp = open
+ ntp.statfiles.open = self.open_jig
+ # Test simplest
+ TDP = self.target.DefaultPeriod
+ faketimemod.time_returns = [TDP * 2]
+ fakeosmod.path.isdir_returns = [True]
+ fakeosmod.path.getmtime_returns = [TDP+1, TDP+2, TDP+3, TDP+4,
+ TDP+5, TDP+6, TDP+7, TDP+8,
+ TDP+9, TDP+10, TDP+11, TDP-1]
+ self.open_returns = [jigs.FileJig(["40594 10\n", "40594 11"]),
+ jigs.FileJig(["40594 30\n", "40594 31"]),
+ jigs.FileJig(["40594 40\n", "40594 41"]),
+ jigs.FileJig(["40594 50\n", "40594 51"]),
+ jigs.FileJig(["40594 60\n", "40594 61"]),
+ jigs.FileJig(["40594 70\n", "40594 71"]),
+ jigs.FileJig(["40594 80\n", "40594 81"]),
+ jigs.FileJig(["604801.25 40594 90\n",
+ "#blah",
+ "604802.25 40594 91"]),
+ jigs.FileJig(["604803.25 40594 100\n",
+ "#blah",
+ "604804.25 40594 101",
+ "604804.25 40595 101"]),
+ jigs.FileJig(["604805.25 40594 110\n",
+ "#blah",
+ "604806.25 40594 111"]),
+ jigs.FileJig(["604807.25 40594 120\n",
+ "#blah",
+ "604808.25 40594 121"])]
+ fakegzipmod.files_returned = [jigs.FileJig(["40594 20\n",
+ "40594 21"])]
+ fakeglobmod.glob_returns = [("/foo/bar/clockstats.0",
+ "/foo/bar/clockstats.1gz"),
+ ("/foo/bar/peerstats.0",
+ "/foo/bar/peerstats.1"),
+ ("/foo/bar/loopstats.0",
+ "/foo/bar/loopstats.1"),
+ ("/foo/bar/rawstats.0",
+ "/foo/bar/rawstats.1"),
+ ("/foo/bar/temps0",
+ "/foo/bar/temps1"),
+ ("/foo/bar/gpsd0",
+ "/foo/bar/gpsd1")] # time kicked
+ cls = self.target("/foo/bar")
+ self.assertEqual(cls.tempssplit(),
+ {"40594": [[604801250, '604801.25',
+ '40594', '90'],
+ [604802250, '604802.25',
+ '40594', '91'],
+ [604803250, '604803.25',
+ '40594', '100'],
+ [604804250, '604804.25',
+ '40594', '101']],
+ "40595": [[604804250, '604804.25',
+ '40595', '101']]})
+ finally:
+ ntp.statfiles.gzip = gziptemp
+ ntp.statfiles.socket = socktemp
+ ntp.statfiles.os = ostemp
+ ntp.statfiles.time = timetemp
+ ntp.statfiles.glob = fakeglobmod
+ ntp.statfiles.open = opentemp
+ sys.stderr = errtemp
+
if __name__ == '__main__':
unittest.main()
View it on GitLab: https://gitlab.com/NTPsec/ntpsec/commit/774f957fa2889f3757157f1a82498610c513ae4b
---
View it on GitLab: https://gitlab.com/NTPsec/ntpsec/commit/774f957fa2889f3757157f1a82498610c513ae4b
You're receiving this email because of your account on gitlab.com.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.ntpsec.org/pipermail/vc/attachments/20170912/7d42fe3c/attachment.html>
More information about the vc
mailing list