From: Robin Jarry Date: Tue, 15 Oct 2019 12:39:17 +0000 (+0200) Subject: usertools: fix pmdinfo with python 3 and pyelftools>=0.24 X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=4da069194ef4578aac7ae10cf05abd992c61723c;p=dpdk.git usertools: fix pmdinfo with python 3 and pyelftools>=0.24 Running dpdk-pmdinfo.py on Ubuntu 18.04 (bionic) with python 3 and pyelftools installed produces no output but no error is reported neither: ~$ python3 usertools/dpdk-pmdinfo.py -r build/app/testpmd ~$ echo $? 0 While with python 2, it works: ~# python2 usertools/dpdk-pmdinfo.py -r build/app/testpmd {"pci_ids": [], "name": "dpio"} {"pci_ids": [], "name": "dpbp"} {"pci_ids": [], "name": "dpaa2_qdma"} ..... On Ubuntu 18.04, pyelftools is version 0.24. The change log of pyelftools v0.24 says: - Symbol/section names are strings internally now, not bytestrings (this may affect API usage in Python 3) (#76). We cannot guess which version of pyelftools is actually being used. The elftools.__version__ symbol is not consistent with each distro's package version. For example, on Ubuntu 16.04 (xenial), the .deb package version is '0.23-2' but elftools.__version__ contains '0.25'. This is certainly due to partial backports. To have a more consistent behaviour of this script across all versions of python, add the unicode_literals future import so that literal strings are now always "unicode". Add 2 utility functions to force a string into bytes or bytes into an unicode string. Force pyelftools return values to unicode strings (will do nothing with recent version of pyelftools). If elffile.get_section_by_name returns None with a unicode section name, try with the same one encoded as bytes. Also, replace all open() calls by io.open() which behaves like the builtin open in python 3. The only non-binary opened file is /usr/share/hwdata/pci.ids which is UTF-8 encoded text. Explicitly specify that encoding. Link: https://github.com/eliben/pyelftools/blob/v0.24/CHANGES#L7 Link: https://github.com/eliben/pyelftools/commit/108eaea9e75a8b5a Fixes: 54ca545dce4b ("make python scripts python2/3 compliant") Cc: stable@dpdk.org Signed-off-by: Robin Jarry Reviewed-by: Olivier Matz --- diff --git a/usertools/dpdk-pmdinfo.py b/usertools/dpdk-pmdinfo.py index 03623d5b8b..069a3bf124 100755 --- a/usertools/dpdk-pmdinfo.py +++ b/usertools/dpdk-pmdinfo.py @@ -8,13 +8,15 @@ # # ------------------------------------------------------------------------- from __future__ import print_function +from __future__ import unicode_literals import json +import io import os import platform import string import sys from elftools.common.exceptions import ELFError -from elftools.common.py3compat import (byte2int, bytes2str, str2bytes) +from elftools.common.py3compat import byte2int from elftools.elf.elffile import ELFFile from optparse import OptionParser @@ -213,7 +215,8 @@ class PCIIds: """ Reads the local file """ - self.contents = open(filename).readlines() + with io.open(filename, 'r', encoding='utf-8') as f: + self.contents = f.readlines() self.date = self.findDate(self.contents) def loadLocal(self): @@ -267,7 +270,13 @@ class ReadElf(object): return None except ValueError: # Not a number. Must be a name then - return self.elffile.get_section_by_name(str2bytes(spec)) + section = self.elffile.get_section_by_name(force_unicode(spec)) + if section is None: + # No match with a unicode name. + # Some versions of pyelftools (<= 0.23) store internal strings + # as bytes. Try again with the name encoded as bytes. + section = self.elffile.get_section_by_name(force_bytes(spec)) + return section def pretty_print_pmdinfo(self, pmdinfo): global pcidb @@ -339,7 +348,8 @@ class ReadElf(object): while endptr < len(data) and byte2int(data[endptr]) != 0: endptr += 1 - mystring = bytes2str(data[dataptr:endptr]) + # pyelftools may return byte-strings, force decode them + mystring = force_unicode(data[dataptr:endptr]) rc = mystring.find("PMD_INFO_STRING") if (rc != -1): self.parse_pmd_info_string(mystring) @@ -348,9 +358,10 @@ class ReadElf(object): def find_librte_eal(self, section): for tag in section.iter_tags(): - if tag.entry.d_tag == 'DT_NEEDED': - if "librte_eal" in tag.needed: - return tag.needed + # pyelftools may return byte-strings, force decode them + if force_unicode(tag.entry.d_tag) == 'DT_NEEDED': + if "librte_eal" in force_unicode(tag.needed): + return force_unicode(tag.needed) return None def search_for_autoload_path(self): @@ -373,7 +384,7 @@ class ReadElf(object): return (None, None) if raw_output is False: print("Scanning for autoload path in %s" % library) - scanfile = open(library, 'rb') + scanfile = io.open(library, 'rb') scanelf = ReadElf(scanfile, sys.stdout) except AttributeError: # Not a dynamic binary @@ -403,7 +414,8 @@ class ReadElf(object): while endptr < len(data) and byte2int(data[endptr]) != 0: endptr += 1 - mystring = bytes2str(data[dataptr:endptr]) + # pyelftools may return byte-strings, force decode them + mystring = force_unicode(data[dataptr:endptr]) rc = mystring.find("DPDK_PLUGIN_PATH") if (rc != -1): rc = mystring.find("=") @@ -416,8 +428,9 @@ class ReadElf(object): def get_dt_runpath(self, dynsec): for tag in dynsec.iter_tags(): - if tag.entry.d_tag == 'DT_RUNPATH': - return tag.runpath + # pyelftools may return byte-strings, force decode them + if force_unicode(tag.entry.d_tag) == 'DT_RUNPATH': + return force_unicode(tag.runpath) return "" def process_dt_needed_entries(self): @@ -438,16 +451,16 @@ class ReadElf(object): return for tag in dynsec.iter_tags(): - if tag.entry.d_tag == 'DT_NEEDED': - rc = tag.needed.find(b"librte_pmd") - if (rc != -1): - library = search_file(tag.needed, + # pyelftools may return byte-strings, force decode them + if force_unicode(tag.entry.d_tag) == 'DT_NEEDED': + if 'librte_pmd' in force_unicode(tag.needed): + library = search_file(force_unicode(tag.needed), runpath + ":" + ldlibpath + ":/usr/lib64:/lib64:/usr/lib:/lib") if library is not None: if raw_output is False: print("Scanning %s for pmd information" % library) - with open(library, 'rb') as file: + with io.open(library, 'rb') as file: try: libelf = ReadElf(file, sys.stdout) except ELFError: @@ -458,6 +471,20 @@ class ReadElf(object): file.close() +# compat: remove force_unicode & force_bytes when pyelftools<=0.23 support is +# dropped. +def force_unicode(s): + if hasattr(s, 'decode') and callable(s.decode): + s = s.decode('latin-1') # same encoding used in pyelftools py3compat + return s + + +def force_bytes(s): + if hasattr(s, 'encode') and callable(s.encode): + s = s.encode('latin-1') # same encoding used in pyelftools py3compat + return s + + def scan_autoload_path(autoload_path): global raw_output @@ -476,7 +503,7 @@ def scan_autoload_path(autoload_path): scan_autoload_path(dpath) if os.path.isfile(dpath): try: - file = open(dpath, 'rb') + file = io.open(dpath, 'rb') readelf = ReadElf(file, sys.stdout) except ELFError: # this is likely not an elf file, skip it @@ -503,7 +530,7 @@ def scan_for_autoload_pmds(dpdk_path): print("Must specify a file name") return - file = open(dpdk_path, 'rb') + file = io.open(dpdk_path, 'rb') try: readelf = ReadElf(file, sys.stdout) except ElfError: @@ -595,7 +622,7 @@ def main(stream=None): print("File not found") sys.exit(1) - with open(myelffile, 'rb') as file: + with io.open(myelffile, 'rb') as file: try: readelf = ReadElf(file, sys.stdout) readelf.process_dt_needed_entries()