From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id A961159CB2 for ; Sat, 16 Apr 2016 18:07:25 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 3607B21C0D3; Sat, 16 Apr 2016 18:06:58 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 760E721C0D0 for ; Sat, 16 Apr 2016 18:06:57 +0000 (UTC) Received: from localhost.localdomain (d202-252.icpnet.pl [109.173.202.252]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-SHA256 (128/128 bits)) (No client certificate requested) (Authenticated sender: mgorny) by smtp.gentoo.org (Postfix) with ESMTPSA id 63D65340B35; Sat, 16 Apr 2016 18:06:55 +0000 (UTC) From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= To: gentoo-portage-dev@lists.gentoo.org Cc: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Subject: [gentoo-portage-dev] [PATCH 3/4] repoman: Use XML Schema for metadata.xml validation Date: Sat, 16 Apr 2016 20:06:44 +0200 Message-Id: <1460830005-20475-4-git-send-email-mgorny@gentoo.org> X-Mailer: git-send-email 2.8.1 In-Reply-To: <1460830005-20475-1-git-send-email-mgorny@gentoo.org> References: <1460830005-20475-1-git-send-email-mgorny@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: 725911ee-cc5c-4aa9-8b20-0ab271cf9943 X-Archives-Hash: 45269cd518ae260f06c7e9bc0a60d3d0 --- pym/repoman/_xml.py | 16 ++++++------- pym/repoman/checks/ebuilds/pkgmetadata.py | 6 ++--- pym/repoman/metadata.py | 39 ++++++++++++++++--------------- pym/repoman/scanner.py | 8 +++---- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index 2661f14..026cd3a 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -12,7 +12,7 @@ from portage import os from portage.output import red from portage.process import find_binary -from repoman.metadata import fetch_metadata_dtd +from repoman.metadata import fetch_metadata_xsd from repoman._subprocess import repoman_getstatusoutput @@ -53,12 +53,12 @@ class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): class XmlLint(object): - def __init__(self, options, repoman_settings, metadata_dtd=None): - self.metadata_dtd = (metadata_dtd or - os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')) + def __init__(self, options, repoman_settings, metadata_xsd=None): + self.metadata_xsd = (metadata_xsd or + os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd')) self.options = options self.repoman_settings = repoman_settings - self._is_capable = metadata_dtd is not None + self._is_capable = metadata_xsd is not None self.binary = None self._check_capable() @@ -70,7 +70,7 @@ class XmlLint(object): print(red("!!! xmllint not found. Can't check metadata.xml.\n")) self._is_capable = False elif not self._is_capable: - if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): + if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings): sys.exit(1) # this can be problematic if xmllint changes their output self._is_capable = True @@ -94,8 +94,8 @@ class XmlLint(object): # xmlint can produce garbage output even on success, so only dump # the ouput when it fails. st, out = repoman_getstatusoutput( - self.binary + " --nonet --noout --dtdvalid %s %s" % ( - portage._shell_quote(self.metadata_dtd), + self.binary + " --nonet --noout --schema %s %s" % ( + portage._shell_quote(self.metadata_xsd), portage._shell_quote( os.path.join(checkdir, "metadata.xml")))) if st != os.EX_OK: diff --git a/pym/repoman/checks/ebuilds/pkgmetadata.py b/pym/repoman/checks/ebuilds/pkgmetadata.py index 74fec69..0e583fd 100644 --- a/pym/repoman/checks/ebuilds/pkgmetadata.py +++ b/pym/repoman/checks/ebuilds/pkgmetadata.py @@ -40,20 +40,20 @@ from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint class PkgMetadata(object): '''Package metadata.xml checks''' - def __init__(self, options, qatracker, repoman_settings, metadata_dtd=None): + def __init__(self, options, qatracker, repoman_settings, metadata_xsd=None): '''PkgMetadata init function @param options: ArgumentParser.parse_known_args(argv[1:]) options @param qatracker: QATracker instance @param repoman_settings: settings instance - @param metadata_dtd: path of metadata.dtd + @param metadata_xsd: path of metadata.xsd ''' self.options = options self.qatracker = qatracker self.repoman_settings = repoman_settings self.musedict = {} self.xmllint = XmlLint(self.options, self.repoman_settings, - metadata_dtd=metadata_dtd) + metadata_xsd=metadata_xsd) def check(self, xpkg, checkdir, checkdirlist, repolevel): '''Performs the checks on the metadata.xml for the package diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index f1fa53a..147f9d0 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -32,8 +32,9 @@ metadata_xml_declaration = '' \ % (metadata_xml_encoding,) metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' +metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this -metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days +metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days def parse_metadata_use(xml_tree): @@ -85,36 +86,36 @@ def parse_metadata_use(xml_tree): return uselist -def fetch_metadata_dtd(metadata_dtd, repoman_settings): +def fetch_metadata_xsd(metadata_xsd, repoman_settings): """ - Fetch metadata.dtd if it doesn't exist or the ctime is older than - metadata_dtd_ctime_interval. + Fetch metadata.xsd if it doesn't exist or the ctime is older than + metadata_xsd_ctime_interval. @rtype: bool @return: True if successful, otherwise False """ must_fetch = True - metadata_dtd_st = None + metadata_xsd_st = None current_time = int(time.time()) try: - metadata_dtd_st = os.stat(metadata_dtd) + metadata_xsd_st = os.stat(metadata_xsd) except EnvironmentError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise del e else: - # Trigger fetch if metadata.dtd mtime is old or clock is wrong. - if abs(current_time - metadata_dtd_st.st_ctime) \ - < metadata_dtd_ctime_interval: + # Trigger fetch if metadata.xsd mtime is old or clock is wrong. + if abs(current_time - metadata_xsd_st.st_ctime) \ + < metadata_xsd_ctime_interval: must_fetch = False if must_fetch: print() print( - "%s the local copy of metadata.dtd " + "%s the local copy of metadata.xsd " "needs to be refetched, doing that now" % green("***")) print() - parsed_url = urlparse(metadata_dtd_uri) + parsed_url = urlparse(metadata_xsd_uri) setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() fcmd = repoman_settings.get(setting) if not fcmd: @@ -124,29 +125,29 @@ def fetch_metadata_dtd(metadata_dtd, repoman_settings): return False destdir = repoman_settings["DISTDIR"] - fd, metadata_dtd_tmp = tempfile.mkstemp( - prefix='metadata.dtd.', dir=destdir) + fd, metadata_xsd_tmp = tempfile.mkstemp( + prefix='metadata.xsd.', dir=destdir) os.close(fd) try: if not portage.getbinpkg.file_get( - metadata_dtd_uri, destdir, fcmd=fcmd, - filename=os.path.basename(metadata_dtd_tmp)): + metadata_xsd_uri, destdir, fcmd=fcmd, + filename=os.path.basename(metadata_xsd_tmp)): logging.error( - "failed to fetch metadata.dtd from '%s'" % metadata_dtd_uri) + "failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri) return False try: portage.util.apply_secpass_permissions( - metadata_dtd_tmp, + metadata_xsd_tmp, gid=portage.data.portage_gid, mode=0o664, mask=0o2) except portage.exception.PortageException: pass - os.rename(metadata_dtd_tmp, metadata_dtd) + os.rename(metadata_xsd_tmp, metadata_xsd) finally: try: - os.unlink(metadata_dtd_tmp) + os.unlink(metadata_xsd_tmp) except OSError: pass diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 36248cb..1384a12 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -82,11 +82,11 @@ class Scanner(object): portage.util.stack_lists([self.categories], incremental=1)) self.categories = self.repo_settings.repoman_settings.categories - metadata_dtd = None + metadata_xsd = None for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees): - path = os.path.join(path, 'metadata/dtd/metadata.dtd') + path = os.path.join(path, 'metadata/xml-schema/metadata.xsd') if os.path.exists(path): - metadata_dtd = path + metadata_xsd = path break self.portdb = repo_settings.portdb @@ -216,7 +216,7 @@ class Scanner(object): self.fetchcheck = FetchChecks( self.qatracker, self.repo_settings, self.portdb, self.vcs_settings) self.pkgmeta = PkgMetadata(self.options, self.qatracker, - self.repo_settings.repoman_settings, metadata_dtd=metadata_dtd) + self.repo_settings.repoman_settings, metadata_xsd=metadata_xsd) self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker) self.use_flag_checks = USEFlagChecks(self.qatracker, uselist) self.keywordcheck = KeywordChecks(self.qatracker, self.options) -- 2.8.1