[Git][NTPsec/ntpsec][master] 2 commits: Full version of the polyglot module
James Browning
gitlab at mg.gitlab.com
Sun Oct 28 20:26:49 UTC 2018
James Browning pushed to branch master at NTPsec / ntpsec
Commits:
71a99d53 by James Browning at 2018-10-28T16:28:15Z
Full version of the polyglot module
- - - - -
91bdd734 by James Browning at 2018-10-28T16:33:41Z
Fix #505 and unblock !805 !798
- - - - -
2 changed files:
- + pylib/poly.py
- wafhelpers/bin_test.py
Changes:
=====================================
pylib/poly.py
=====================================
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: BSD-2-clause
+"""Handle bytes and strings in a polyglot fashion.
+
+copied from ../ntpclient/ntpq.py which got it from
+https://gitlab.com/esr/practical-python-porting/blob/master/polystr-inclusion.py
+see http://www.catb.org/esr/faqs/practical-python-porting/ for more information.
+"""
+import sys
+import ntp.util
+
+master_encoding = 'latin-1'
+
+# General notes on Python 2/3 compatibility:
+#
+# This code uses the following strategy to allow it to run on both Python 2
+# and Python 3:
+#
+# - Use latin-1 encoding to transform binary data to/from Unicode when
+# necessary for operations where Python 3 expects Unicode; the
+# polystr and polybytes functions are used to do this so that
+# when running on Python 2, the byte string data is used unchanged.
+#
+# - Construct custom stdout and stderr streams when running
+# on Python 3 that force UTF-8 encoding, and wrap them around the
+# underlying binary buffers (in Python 2, the streams are binary
+# and are used unchanged); this ensures that the same transformation
+# is done on data from/to the standard streams, as is done on binary
+# data from/to files and subprocesses; the make_std_wrapper function
+# does this.
+#
+# anyone that changes this needs to test with all combinations of
+# python2, python3, LC_ALL=ascii, LC_ALL=latin-1, LC_ALL=en_US.utf8, and
+# piping output to a file. While looking at the UTF-8 in the output.
+
+forced_utf8 = False
+
+if str is bytes: # Python 2
+ polystr = unicode
+ polybytes = bytes
+ polyord = ord
+ polychr = str
+ input = raw_input
+
+ def string_escape(s):
+ """String_escape/unicode_escape."""
+ return s.decode('string_escape')
+
+ # This used to force UTF-8 encoding, but that breaks the readline system.
+ # Unfortunately sometimes sys.stdout.encoding lies about the encoding,
+ # so expect random false positives.
+ ntp.util.check_unicode()
+
+else: # Python 3
+ import io
+
+ def polystr(o):
+ """Polymorphic string factory function."""
+ if isinstance(o, str):
+ return o
+ if not isinstance(o, bytes):
+ return str(o)
+ return str(o, encoding=master_encoding)
+
+ def polybytes(s):
+ """Polymorphic string encoding function."""
+ if isinstance(s, bytes):
+ return s
+ if not isinstance(s, str):
+ return bytes(s)
+ return bytes(s, encoding=master_encoding)
+
+ def polyord(c):
+ "Polymorphic ord() function"
+ if isinstance(c, str):
+ return ord(c)
+ else:
+ return c
+
+ def polychr(c):
+ "Polymorphic chr() function"
+ if isinstance(c, int):
+ return chr(c)
+ else:
+ return c
+
+ def string_escape(s):
+ """Polymorphic string_escape/unicode_escape."""
+ # This hack is necessary because Unicode strings in Python 3 don't
+ # have a decode method, so there's no simple way to ask it for the
+ # equivalent of decode('string_escape') in Python 2. This function
+ # assumes that it will be called with a Python 3 'str' instance
+ return s.encode(master_encoding).decode('unicode_escape')
+
+ def make_std_wrapper(stream):
+ """Standard input/output wrapper factory function."""
+ # This ensures that the encoding of standard output and standard
+ # error on Python 3 matches the master encoding we use to turn
+ # bytes to Unicode in polystr above
+ # line_buffering=True ensures that interactive command sessions
+ # work as expected
+ return io.TextIOWrapper(stream.buffer, encoding="utf-8",
+ newline="\n", line_buffering=True)
+
+ # This is the one situation where we *can* force unicode.
+ if "UTF-8" != sys.stdout.encoding:
+ forced_utf8 = True
+ sys.stdout = make_std_wrapper(sys.stdout)
+ if "UTF-8" != sys.stderr.encoding:
+ forced_utf8 = True
+ sys.stderr = make_std_wrapper(sys.stderr)
=====================================
wafhelpers/bin_test.py
=====================================
@@ -1,73 +1,86 @@
+"""Run a suite of tests on the listed binaries."""
from __future__ import print_function
+import os
+import os.path
import sys
-from os.path import exists
-from waflib.Utils import subprocess
-from waflib.Logs import pprint
+import waflib.Context
+import waflib.Logs
+import waflib.Utils
+sys.path.insert(0, "%s/main/tests/pylib" % waflib.Context.out_dir)
+import ntp.poly
+import ntp.util
-# Need the build form of util.py to get the version string
-sys.path.insert(0, "build/main/pylib/")
-import util
-
-verStr = util.stdversion()
+verStr = ntp.util.stdversion()
cmd_map = {
+ ("main/ntpclients/ntpleapfetch", "--version"): "ntpleapfetch %s\n"
+ % verStr,
("main/ntpd/ntpd", "--version"): "ntpd %s\n" % verStr,
+ ("main/ntpfrob/ntpfrob", "-V"): "ntpfrob %s\n" % verStr,
+ ("main/ntptime/ntptime", "-V"): "ntptime %s\n" % verStr
+}
+cmd_map2 = {
("main/ntpclients/ntpdig", "--version"): "ntpdig %s\n" % verStr,
("main/ntpclients/ntpkeygen", "--version"): "ntpkeygen %s\n" % verStr,
("main/ntpclients/ntpq", "--version"): "ntpq %s\n" % verStr,
("main/ntpclients/ntpmon", "--version"): "ntpmon %s\n" % verStr,
- ("main/ntpclients/ntpleapfetch", "--version"): "ntpleapfetch %s\n" % verStr,
("main/ntpclients/ntplogtemp", "--version"): "ntplogtemp %s\n" % verStr,
("main/ntpclients/ntpsnmpd", "--version"): "ntpsnmpd %s\n" % verStr,
("main/ntpclients/ntpsweep", "--version"): "ntpsweep %s\n" % verStr,
("main/ntpclients/ntptrace", "--version"): "ntptrace %s\n" % verStr,
("main/ntpclients/ntpviz", "--version"): "ntpviz %s\n" % verStr,
- ("main/ntpclients/ntpwait", "--version"): "ntpwait %s\n" % verStr,
- ("main/ntpfrob/ntpfrob", "-V"): "ntpfrob %s\n" % verStr,
- ("main/ntptime/ntptime", "-V"): "ntptime %s\n" % verStr,
+ ("main/ntpclients/ntpwait", "--version"): "ntpwait %s\n" % verStr
}
-# XXX: Needs to run in a thread with a timeout.
-def run(cmd, reg):
+def run(cmd, reg, pythonic):
+ """Run an individual non-python test."""
check = False
- if cmd[1] is None:
- cmd = [cmd[0]]
+ breg = ntp.poly.polybytes(reg)
print("running: ", " ".join(cmd), end="")
- if not exists("build/%s" % cmd[0]):
- pprint("YELLOW", " SKIPPING (does not exist)")
+ if not os.path.exists("%s/%s" % (waflib.Context.out_dir, cmd[0])):
+ waflib.Logs.pprint("YELLOW", " SKIPPING (does not exist)")
return False
- p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=None, cwd="build")
+ if pythonic:
+ cmd = [sys.executable] + list(cmd)
+ p = waflib.Utils.subprocess.Popen(cmd, env={'PYTHONPATH': '%s/main/tests/pylib' %
+ waflib.Context.out_dir},
+ stdin=waflib.Utils.subprocess.PIPE,
+ stdout=waflib.Utils.subprocess.PIPE,
+ stderr=waflib.Utils.subprocess.PIPE, cwd=waflib.Context.out_dir)
stdout, stderr = p.communicate()
- if (stdout == reg) or (stderr == reg):
+ if (stdout == breg) or (stderr == breg):
check = True
if check:
- pprint("GREEN", " OK")
+ waflib.Logs.pprint("GREEN", " OK")
return True
- else:
- pprint("RED", " FAILED")
- return False
+ waflib.Logs.pprint("RED", " FAILED")
+ waflib.Logs.pprint("PINK", ntp.poly.polystr(stderr))
+ return False
def cmd_bin_test(ctx, config):
+ """Run a suite of binary tests."""
fails = 0
for cmd in sorted(cmd_map):
- if not run(cmd, cmd_map[cmd]):
+ if not run(cmd, cmd_map[cmd], False):
+ fails += 1
+
+ for cmd in sorted(cmd_map2):
+ if not run(cmd, cmd_map2[cmd], True):
fails += 1
+ if 0 < fails:
+ waflib.Logs.pprint("GREY", "Expected:\t%s" % (verStr))
if 1 == fails:
ctx.fatal("1 binary test failed!")
elif 1 < fails:
ctx.fatal("%d binary tests failed!" % fails)
-
-# cmd_bin_test(None, None)
View it on GitLab: https://gitlab.com/NTPsec/ntpsec/compare/b4120c5c40aeae5d5a52bd62f38831c0f6b958be...91bdd7341286a50484f34274ff832a5c226eae8e
--
View it on GitLab: https://gitlab.com/NTPsec/ntpsec/compare/b4120c5c40aeae5d5a52bd62f38831c0f6b958be...91bdd7341286a50484f34274ff832a5c226eae8e
You're receiving this email because of your account on gitlab.com.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.ntpsec.org/pipermail/vc/attachments/20181028/cc362597/attachment-0001.html>
More information about the vc
mailing list