[gentoo-portage-dev] [PATCH v2 1/4] Support post-postinst QA checks

[gentoo-portage-dev] [PATCH v2 1/4] Support post-postinst QA checks

[Michał Górny]

Extend the QA check mechanics in Portage to support post-postinst QA
checks. They are like post-install QA checks, except they are run after
pkg_postinst(), and so they can be used to verify that necessary
postinst actions were performed (e.g. regenerating caches).
---
 bin/misc-functions.sh                  | 57 ++++++++++++++++++++++++++++++++++
 pym/portage/package/ebuild/doebuild.py |  5 ++-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 079369313..18cddea21 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -256,6 +256,63 @@ install_qa_check() {
 	rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
 }
 
+postinst_qa_check() {
+	local d f paths qa_checks=()
+	if ! ___eapi_has_prefix_variables; then
+		local EPREFIX= EROOT=${ROOT}
+	fi
+
+	cd "${EROOT}" || die "cd failed"
+
+	# Collect the paths for QA checks, highest prio first.
+	paths=(
+		# sysadmin overrides
+		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/local/lib/postinst-qa-check.d
+		# system-wide package installs
+		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/lib/postinst-qa-check.d
+	)
+
+	# Now repo-specific checks.
+	# (yes, PORTAGE_ECLASS_LOCATIONS contains repo paths...)
+	for d in "${PORTAGE_ECLASS_LOCATIONS[@]}"; do
+		paths+=(
+			"${d}"/metadata/postinst-qa-check.d
+		)
+	done
+
+	paths+=(
+		# Portage built-in checks
+		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/lib/portage/postinst-qa-check.d
+		"${PORTAGE_BIN_PATH}"/postinst-qa-check.d
+	)
+
+	# Collect file names of QA checks. We need them early to support
+	# overrides properly.
+	for d in "${paths[@]}"; do
+		for f in "${d}"/*; do
+			[[ -f ${f} ]] && qa_checks+=( "${f##*/}" )
+		done
+	done
+
+	# Now we need to sort the filenames lexically, and process
+	# them in order.
+	while read -r -d '' f; do
+		# Find highest priority file matching the basename.
+		for d in "${paths[@]}"; do
+			[[ -f ${d}/${f} ]] && break
+		done
+
+		# Run in a subshell to treat it like external script,
+		# but use 'source' to pass all variables through.
+		(
+			# Allow inheriting eclasses.
+			# XXX: we want this only in repository-wide checks.
+			_IN_INSTALL_QA_CHECK=1
+			source "${d}/${f}" || eerror "Post-postinst QA check ${f} failed to run"
+		)
+	done < <(printf "%s\0" "${qa_checks[@]}" | LC_ALL=C sort -u -z)
+}
+
 install_mask() {
 	local root="$1"
 	shift
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index 14d96f57c..ac697a763 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1738,7 +1738,10 @@ _post_phase_cmds = {
 		"preinst_sfperms",
 		"preinst_selinux_labels",
 		"preinst_suid_scan",
-		]
+		],
+
+	"postinst" : [
+		"postinst_qa_check"],
 }
 
 def _post_phase_userpriv_perms(mysettings):
-- 
2.14.1

[gentoo-portage-dev] [PATCH v2 2/4] Add post-postinst checks for a few missed cache updates

[Michał Górny]

Add postinst-qa-check.d checks for missed desktop, mime-info and GTK+
icon cache updates. In all of the cases the checks simply look for any
installed files that are newer than the cache.

This check has some limitations: it assumes that mtime is not preserved
when copying files to D, it can't distinguish whether the files
were installed by the current package (it reports all new files since
the last cache update) and it can't distinguish between the update
on postinst and postrm. However, it's certainly a step forward and will
help find a few bugs.
---
 bin/postinst-qa-check.d/50gnome2-utils | 38 ++++++++++++++++++++
 bin/postinst-qa-check.d/50xdg-utils    | 65 ++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 bin/postinst-qa-check.d/50gnome2-utils
 create mode 100644 bin/postinst-qa-check.d/50xdg-utils

diff --git a/bin/postinst-qa-check.d/50gnome2-utils b/bin/postinst-qa-check.d/50gnome2-utils
new file mode 100644
index 000000000..68e21cb74
--- /dev/null
+++ b/bin/postinst-qa-check.d/50gnome2-utils
@@ -0,0 +1,38 @@
+# check for missing calls to gnome2-utils regen functions
+
+gnome2_icon_cache_check() {
+	local d f files=() find_args
+	for d in usr/share/icons/*/; do
+		# gnome2_icon_cache_update updates only themes with an index
+		[[ -f ${d}/index.theme ]] || continue
+
+		find_args=()
+		# if the cache does not exist at all, we complain for any file
+		# otherwise, we look for files newer than the cache
+		[[ -f ${d}/icon-theme.cache ]] &&
+			find_args+=( -newer "${d}"/icon-theme.cache )
+
+		# (use -mindepth 2 to easily skip the cache files)
+		while read -r -d $'\0' f; do
+			files+=( "${f}" )
+		done < <(find "${d}" -mindepth 2 -type f "${find_args[@]}" -print0)
+	done
+
+	if [[ ${files[@]} ]]; then
+		eqawarn "QA Notice: new icons were found installed but GTK+ icon cache"
+		eqawarn "has not been updated:"
+		eqatag -v gnome2-utils.icon-cache "${files[@]/#//}"
+		eqawarn "Please make sure to call gnome2_icon_cache_update()"
+		eqawarn "in pkg_postinst() and pkg_postrm() phases of appropriate pkgs."
+	fi
+}
+
+gnome2_utils_postinst_check() {
+	cd "${EROOT}" || die
+	gnome2_icon_cache_check
+}
+
+gnome2_utils_postinst_check
+: # guarantee successful exit
+
+# vim:ft=sh
diff --git a/bin/postinst-qa-check.d/50xdg-utils b/bin/postinst-qa-check.d/50xdg-utils
new file mode 100644
index 000000000..4bc7bee9a
--- /dev/null
+++ b/bin/postinst-qa-check.d/50xdg-utils
@@ -0,0 +1,65 @@
+# check for missing calls to xdg-utils regen functions
+
+xdg_desktop_database_check() {
+	local d f files=()
+	for d in usr/share/applications; do
+		[[ -d ${d} ]] || continue
+
+		find_args=()
+		# if the cache does not exist at all, we complain for any file
+		# otherwise, we look for files newer than the cache
+		[[ -f ${d}/mimeinfo.cache ]] &&
+			find_args+=( -newer "${d}"/mimeinfo.cache )
+
+		# look for any .desktop files that are newer than the cache
+		# and that have any mime types defined
+		while read -r -d $'\0' f; do
+			files+=( "${f}" )
+		done < <(find "${d}" -name '*.desktop' "${find_args[@]}" \
+			-exec grep -lZi '^MimeType=' {} +)
+	done
+
+	if [[ ${files[@]} ]]; then
+		eqawarn "QA Notice: .desktop files with MimeType= were found installed"
+		eqawarn "but desktop mimeinfo cache has not been updated:"
+		eqatag -v xdg-utils.desktop "${files[@]/#//}"
+		eqawarn "Please make sure to call xdg_desktop_database_update()"
+		eqawarn "in pkg_postinst() and pkg_postrm() phases of appropriate pkgs."
+	fi
+}
+
+xdg_mimeinfo_database_check() {
+	local d f files=()
+	for d in usr/share/mime; do
+		[[ -d ${d} ]] || continue
+
+		find_args=()
+		# if the cache does not exist at all, we complain for any file
+		# otherwise, we look for files newer than the cache
+		[[ -f ${d}/mime.cache ]] &&
+			find_args+=( -newer "${d}"/mime.cache )
+
+		while read -r -d $'\0' f; do
+			files+=( "${f}" )
+		done < <(find "${d}" -name '*.xml' "${find_args[@]}" -print0)
+	done
+
+	if [[ ${files[@]} ]]; then
+		eqawarn "QA Notice: mime-info files were found installed but mime-info"
+		eqawarn "cache has not been updated:"
+		eqatag -v xdg-utils.mime-info "${files[@]/#//}"
+		eqawarn "Please make sure to call xdg_mimeinfo_database_update()"
+		eqawarn "in pkg_postinst() and pkg_postrm() phases of appropriate pkgs."
+	fi
+}
+
+xdg_utils_postinst_check() {
+	cd "${EROOT}" || die
+	xdg_desktop_database_check
+	xdg_mimeinfo_database_check
+}
+
+xdg_utils_postinst_check
+: # guarantee successful exit
+
+# vim:ft=sh
-- 
2.14.1

[gentoo-portage-dev] [PATCH v2 3/4] Use ctime in cache post-postinst checks

[Michał Górny]

Use ctime rather than mtime in cache post-postinst checks since mtime
may be preserved from the original tarball, and therefore be 'long time
ago'. ctime is more reliable in this regard.
---
 bin/postinst-qa-check.d/50gnome2-utils | 2 +-
 bin/postinst-qa-check.d/50xdg-utils    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/postinst-qa-check.d/50gnome2-utils b/bin/postinst-qa-check.d/50gnome2-utils
index 68e21cb74..4e7c6eb85 100644
--- a/bin/postinst-qa-check.d/50gnome2-utils
+++ b/bin/postinst-qa-check.d/50gnome2-utils
@@ -10,7 +10,7 @@ gnome2_icon_cache_check() {
 		# if the cache does not exist at all, we complain for any file
 		# otherwise, we look for files newer than the cache
 		[[ -f ${d}/icon-theme.cache ]] &&
-			find_args+=( -newer "${d}"/icon-theme.cache )
+			find_args+=( -newercm "${d}"/icon-theme.cache )
 
 		# (use -mindepth 2 to easily skip the cache files)
 		while read -r -d $'\0' f; do
diff --git a/bin/postinst-qa-check.d/50xdg-utils b/bin/postinst-qa-check.d/50xdg-utils
index 4bc7bee9a..b0bb029bd 100644
--- a/bin/postinst-qa-check.d/50xdg-utils
+++ b/bin/postinst-qa-check.d/50xdg-utils
@@ -9,7 +9,7 @@ xdg_desktop_database_check() {
 		# if the cache does not exist at all, we complain for any file
 		# otherwise, we look for files newer than the cache
 		[[ -f ${d}/mimeinfo.cache ]] &&
-			find_args+=( -newer "${d}"/mimeinfo.cache )
+			find_args+=( -newercm "${d}"/mimeinfo.cache )
 
 		# look for any .desktop files that are newer than the cache
 		# and that have any mime types defined
@@ -37,7 +37,7 @@ xdg_mimeinfo_database_check() {
 		# if the cache does not exist at all, we complain for any file
 		# otherwise, we look for files newer than the cache
 		[[ -f ${d}/mime.cache ]] &&
-			find_args+=( -newer "${d}"/mime.cache )
+			find_args+=( -newercm "${d}"/mime.cache )
 
 		while read -r -d $'\0' f; do
 			files+=( "${f}" )
-- 
2.14.1

[gentoo-portage-dev] [PATCH v2 4/4] Update caches after reporting missing cache updates

[Michał Górny]

Call the appropriate updater to update caches after reporting a missing
cache update, in order to stop repeating the same issue on subsequent
packages that did not install any relevant files.
---
 bin/postinst-qa-check.d/50gnome2-utils |  9 +++++++++
 bin/postinst-qa-check.d/50xdg-utils    | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/bin/postinst-qa-check.d/50gnome2-utils b/bin/postinst-qa-check.d/50gnome2-utils
index 4e7c6eb85..d0cbb4037 100644
--- a/bin/postinst-qa-check.d/50gnome2-utils
+++ b/bin/postinst-qa-check.d/50gnome2-utils
@@ -16,6 +16,15 @@ gnome2_icon_cache_check() {
 		while read -r -d $'\0' f; do
 			files+=( "${f}" )
 		done < <(find "${d}" -mindepth 2 -type f "${find_args[@]}" -print0)
+
+		# if any files were found, update the db to avoid repeating
+		# the warning for subsequent packages
+		# (note: yes, it will eagerly repeat the update for next dirs
+		# but that's a minor issue)
+		if [[ ${files[@]} ]]; then
+			addwrite "${d}"
+			gtk-update-icon-cache -qf "${d}"
+		fi
 	done
 
 	if [[ ${files[@]} ]]; then
diff --git a/bin/postinst-qa-check.d/50xdg-utils b/bin/postinst-qa-check.d/50xdg-utils
index b0bb029bd..9f5e9a48c 100644
--- a/bin/postinst-qa-check.d/50xdg-utils
+++ b/bin/postinst-qa-check.d/50xdg-utils
@@ -17,6 +17,15 @@ xdg_desktop_database_check() {
 			files+=( "${f}" )
 		done < <(find "${d}" -name '*.desktop' "${find_args[@]}" \
 			-exec grep -lZi '^MimeType=' {} +)
+
+		# if any files were found, update the db to avoid repeating
+		# the warning for subsequent packages
+		# (note: yes, it will eagerly repeat the update for next dirs
+		# but it's a minor issue and we have only one dir anyway)
+		if [[ ${files[@]} ]]; then
+			addwrite "${d}"
+			update-desktop-database "${d}"
+		fi
 	done
 
 	if [[ ${files[@]} ]]; then
@@ -42,6 +51,15 @@ xdg_mimeinfo_database_check() {
 		while read -r -d $'\0' f; do
 			files+=( "${f}" )
 		done < <(find "${d}" -name '*.xml' "${find_args[@]}" -print0)
+
+		# if any files were found, update the db to avoid repeating
+		# the warning for subsequent packages
+		# (note: yes, it will eagerly repeat the update for next dirs
+		# but it's a minor issue and we have only one dir anyway)
+		if [[ ${files[@]} ]]; then
+			addwrite "${d}"
+			update-mime-database "${d}"
+		fi
 	done
 
 	if [[ ${files[@]} ]]; then
-- 
2.14.1

Re: [gentoo-portage-dev] [PATCH v2 1/4] Support post-postinst QA checks

[Michał Górny]

W dniu śro, 16.08.2017 o godzinie 20∶09 +0200, użytkownik Michał Górny
napisał:
> Extend the QA check mechanics in Portage to support post-postinst QA
> checks. They are like post-install QA checks, except they are run after
> pkg_postinst(), and so they can be used to verify that necessary
> postinst actions were performed (e.g. regenerating caches).
> ---
>  bin/misc-functions.sh                  | 57 ++++++++++++++++++++++++++++++++++
>  pym/portage/package/ebuild/doebuild.py |  5 ++-
>  2 files changed, 61 insertions(+), 1 deletion(-)
> 
> diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
> index 079369313..18cddea21 100755
> --- a/bin/misc-functions.sh
> +++ b/bin/misc-functions.sh
> @@ -256,6 +256,63 @@ install_qa_check() {
>  	rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
>  }
>  
> +postinst_qa_check() {
> +	local d f paths qa_checks=()
> +	if ! ___eapi_has_prefix_variables; then
> +		local EPREFIX= EROOT=${ROOT}
> +	fi
> +
> +	cd "${EROOT}" || die "cd failed"
> +
> +	# Collect the paths for QA checks, highest prio first.
> +	paths=(
> +		# sysadmin overrides
> +		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/local/lib/postinst-qa-check.d
> +		# system-wide package installs
> +		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/lib/postinst-qa-check.d
> +	)
> +
> +	# Now repo-specific checks.
> +	# (yes, PORTAGE_ECLASS_LOCATIONS contains repo paths...)
> +	for d in "${PORTAGE_ECLASS_LOCATIONS[@]}"; do
> +		paths+=(
> +			"${d}"/metadata/postinst-qa-check.d
> +		)
> +	done
> +
> +	paths+=(
> +		# Portage built-in checks
> +		"${PORTAGE_OVERRIDE_EPREFIX}"/usr/lib/portage/postinst-qa-check.d
> +		"${PORTAGE_BIN_PATH}"/postinst-qa-check.d
> +	)
> +
> +	# Collect file names of QA checks. We need them early to support
> +	# overrides properly.
> +	for d in "${paths[@]}"; do
> +		for f in "${d}"/*; do
> +			[[ -f ${f} ]] && qa_checks+=( "${f##*/}" )
> +		done
> +	done
> +
> +	# Now we need to sort the filenames lexically, and process
> +	# them in order.
> +	while read -r -d '' f; do
> +		# Find highest priority file matching the basename.
> +		for d in "${paths[@]}"; do
> +			[[ -f ${d}/${f} ]] && break
> +		done
> +
> +		# Run in a subshell to treat it like external script,
> +		# but use 'source' to pass all variables through.
> +		(
> +			# Allow inheriting eclasses.
> +			# XXX: we want this only in repository-wide checks.
> +			_IN_INSTALL_QA_CHECK=1
> +			source "${d}/${f}" || eerror "Post-postinst QA check ${f} failed to run"
> +		)
> +	done < <(printf "%s\0" "${qa_checks[@]}" | LC_ALL=C sort -u -z)
> +}
> +
>  install_mask() {
>  	local root="$1"
>  	shift
> diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
> index 14d96f57c..ac697a763 100644
> --- a/pym/portage/package/ebuild/doebuild.py
> +++ b/pym/portage/package/ebuild/doebuild.py
> @@ -1738,7 +1738,10 @@ _post_phase_cmds = {
>  		"preinst_sfperms",
>  		"preinst_selinux_labels",
>  		"preinst_suid_scan",
> -		]
> +		],
> +
> +	"postinst" : [
> +		"postinst_qa_check"],
>  }
>  
>  def _post_phase_userpriv_perms(mysettings):

Merged now.

-- 
Best regards,
Michał Górny