[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     2df85e576c305cd00f93ef27107218808a4e5cef
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  1 12:58:27 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar  1 12:58:27 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=2df85e57

scripts/rsync-generation/hashgen: always use sorted directory listing

To get reproducible outputs, ensure we sort the directory listings, such
that the order and thus optional compression ratio is the same.  This is
in particular nice when multiple generators are in use.

 scripts/rsync-generation/hashgen.c | 214 +++++++++++++++++++++----------------
 1 file changed, 122 insertions(+), 92 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 534d278d91..93723f7e60 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -203,6 +203,65 @@ get_hashes(
 	}
 }
 
+#define LISTSZ 64
+
+static int
+compare_strings(const void *l, const void *r)
+{
+	const char **strl = (const char **)l;
+	const char **strr = (const char **)r;
+	return strcmp(*strl, *strr);
+}
+
+/**
+ * Return a sorted list of entries in the given directory.  All entries
+ * starting with a dot are ignored, and not present in the returned
+ * list.  The list and all entries are allocated using malloc() and need
+ * to be freed.
+ */
+static char
+list_dir(char ***retlist, size_t *retcnt, const char *path)
+{
+	DIR *d;
+	struct dirent *e;
+	size_t rlen = 0;
+	size_t rsize = 0;
+	char **rlist = NULL;
+
+	if ((d = opendir(path)) != NULL) {
+		while ((e = readdir(d)) != NULL) {
+			/* skip all dotfiles */
+			if (e->d_name[0] == '.')
+				continue;
+
+			if (rlen == rsize) {
+				rsize += LISTSZ;
+				rlist = realloc(rlist,
+						rsize * sizeof(rlist[0]));
+				if (rlist == NULL) {
+					fprintf(stderr, "out of memory\n");
+					return 1;
+				}
+			}
+			rlist[rlen] = strdup(e->d_name);
+			if (rlist[rlen] == NULL) {
+				fprintf(stderr, "out of memory\n");
+				return 1;
+			}
+			rlen++;
+		}
+		closedir(d);
+
+		qsort(rlist, rlen, sizeof(rlist[0]), compare_strings);
+
+		*retlist = rlist;
+		*retcnt = rlen;
+		return 0;
+	} else {
+		return 1;
+	}
+}
+
 static void
 write_hashes(
 		struct timeval *tv,
@@ -260,25 +319,24 @@ write_hashes_dir(
 		gzFile zm)
 {
 	char path[8192];
-	DIR *d;
-	struct dirent *e;
+	char **dentries;
+	size_t dentrieslen;
+	size_t i;
 
 	snprintf(path, sizeof(path), "%s/%s", root, name);
-	if ((d = opendir(path)) != NULL) {
-		while ((e = readdir(d)) != NULL) {
-			/* skip all dotfiles */
-			if (e->d_name[0] == '.')
-				continue;
-			snprintf(path, sizeof(path), "%s/%s", name, e->d_name);
-			if (write_hashes_dir(tv, root, path, zm))
+	if (list_dir(&dentries, &dentrieslen, path) != 0) {
+		for (i = 0; i < dentrieslen; i++) {
+			snprintf(path, sizeof(path), "%s/%s", name, dentries[i]);
+			free(dentries[i]);
+			if (write_hashes_dir(tv, root, path, zm) == 0)
 				continue;
 			/* regular file */
 			write_hashes(tv, root, path, "DATA", NULL, zm);
 		}
-		closedir(d);
-		return 1;
-	} else {
+		free(dentries);
 		return 0;
+	} else {
+		return 1;
 	}
 }
 
@@ -286,27 +344,26 @@ static char
 process_files(const char *dir, const char *off, FILE *m)
 {
 	char path[8192];
-	DIR *d;
-	struct dirent *e;
+	char **dentries;
+	size_t dentrieslen;
+	size_t i;
 	struct timeval tv[2]; /* dummy, won't use its result */
 
 	snprintf(path, sizeof(path), "%s/%s", dir, off);
-	if ((d = opendir(path)) != NULL) {
-		while ((e = readdir(d)) != NULL) {
-			/* skip all dotfiles */
-			if (e->d_name[0] == '.')
-				continue;
+	if (list_dir(&dentries, &dentrieslen, path) != 0) {
+		for (i = 0; i < dentrieslen; i++) {
 			snprintf(path, sizeof(path), "%s%s%s",
-					off, *off == '\0' ? "" : "/", e->d_name);
-			if (process_files(dir, path, m))
+					off, *off == '\0' ? "" : "/", dentries[i]);
+			free(dentries[i]);
+			if (process_files(dir, path, m) == 0)
 				continue;
 			/* regular file */
 			write_hashes(tv, dir, path, "AUX", m, NULL);
 		}
-		closedir(d);
-		return 1;
-	} else {
+		free(dentries);
 		return 0;
+	} else {
+		return 1;
 	}
 }
 
@@ -399,8 +456,6 @@ process_dir_gen(const char *dir)
 {
 	char manifest[8192];
 	FILE *f;
-	DIR *d;
-	struct dirent *e;
 	char path[8192];
 	const char *p;
 	int newhashes;
@@ -460,8 +515,11 @@ process_dir_gen(const char *dir)
 		/* all of these types (GLOBAL, SUBTREE, CATEGORY) have a gzipped
 		 * Manifest */
 		gzFile mf;
+		char **dentries;
+		size_t dentrieslen;
+		size_t i;
 
-		if ((d = opendir(dir)) != NULL) {
+		if (list_dir(&dentries, &dentrieslen, dir) != 0) {
 			char *my_manifest = str_manifest_gz;
 
 			if (type_manifest == GLOBAL_MANIFEST)
@@ -474,22 +532,21 @@ process_dir_gen(const char *dir)
 				return NULL;
 			}
 
-			while ((e = readdir(d)) != NULL) {
-				/* ignore dotfiles (including . and ..) */
-				if (e->d_name[0] == '.')
-					continue;
+			for (i = 0; i < dentrieslen; i++) {
 				/* ignore existing Manifests */
-				if (strcmp(e->d_name, my_manifest) == 0)
-					continue;
-				if (strcmp(e->d_name, str_manifest) == 0)
+				if (strcmp(dentries[i], my_manifest) == 0 ||
+						strcmp(dentries[i], str_manifest) == 0)
+				{
+					free(dentries[i]);
 					continue;
+				}
 
-				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
+				snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
 				if (!stat(path, &s)) {
 					if (s.st_mode & S_IFDIR) {
 						if (type_manifest == SUBTREE_MANIFEST) {
-							write_hashes_dir(tv, dir, e->d_name, mf);
-							if (strcmp(e->d_name, "metadata") == 0) {
+							write_hashes_dir(tv, dir, dentries[i], mf);
+							if (strcmp(dentries[i], "metadata") == 0) {
 								char buf[2048];
 								size_t len;
 								len = snprintf(buf, sizeof(buf),
@@ -499,22 +556,26 @@ process_dir_gen(const char *dir)
 										"IGNORE timestamp.x\n");
 								gzwrite(mf, buf, len);
 							}
+							free(dentries[i]);
 						} else {
 							char *mfest = process_dir_gen(path);
 							if (mfest == NULL) {
 								gzclose(mf);
+								free(dentries[i]);
 								return NULL;
 							}
 							snprintf(path, sizeof(path), "%s/%s",
-									e->d_name, mfest);
+									dentries[i], mfest);
+							free(dentries[i]);
 							write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
 						}
 					} else if (s.st_mode & S_IFREG) {
-						write_hashes(tv, dir, e->d_name, "DATA", NULL, mf);
+						write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
+						free(dentries[i]);
 					}
 				}
 			}
-			closedir(d);
+			free(dentries);
 
 			if (type_manifest == GLOBAL_MANIFEST) {
 				char globmanifest[8192];
@@ -566,6 +627,9 @@ process_dir_gen(const char *dir)
 		FILE *m;
 		char newmanifest[8192];
 		char buf[8192];
+		char **dentries;
+		size_t dentrieslen;
+		size_t i;
 
 		snprintf(newmanifest, sizeof(newmanifest), "%s/.Manifest.new", dir);
 		if ((m = fopen(newmanifest, "w")) == NULL) {
@@ -593,18 +657,18 @@ process_dir_gen(const char *dir)
 		}
 		fclose(f);
 
-		if ((d = opendir(dir)) != NULL) {
-			while ((e = readdir(d)) != NULL) {
-				/* in ebuild land, stuff starting with a . isn't valid,
-				 * so can safely ignore it, while at the same time
-				 * skipping over . and .. (+need for .Manifest.new) */
-				if (e->d_name[0] == '.')
-					continue;
-				if (strcmp(e->d_name + strlen(e->d_name) - 7, ".ebuild") != 0)
+		if (list_dir(&dentries, &dentrieslen, dir) != 0) {
+			for (i = 0; i < dentrieslen; i++) {
+				if (strcmp(dentries[i] + strlen(dentries[i]) - 7,
+							".ebuild") != 0)
+				{
+					free(dentries[i]);
 					continue;
-				write_hashes(tv, dir, e->d_name, "EBUILD", m, NULL);
+				}
+				write_hashes(tv, dir, dentries[i], "EBUILD", m, NULL);
+				free(dentries[i]);
 			}
-			closedir(d);
+			free(dentries);
 		}
 
 		write_hashes(tv, dir, "ChangeLog", "MISC", m, NULL);
@@ -915,23 +979,14 @@ compare_elems(const void *l, const void *r)
 	return cl - cr;
 }
 
-static int
-compare_strings(const void *l, const void *r)
-{
-	const char **strl = (const char **)l;
-	const char **strr = (const char **)r;
-	return strcmp(*strl, *strr);
-}
-
-static char verify_manifest(const char *dir, const char *manifest);
-
 struct subdir_workload {
 	size_t subdirlen;
 	size_t elemslen;
 	char **elems;
 };
 
-#define LISTSZ 64
+static char verify_manifest(const char *dir, const char *manifest);
+
 static char
 verify_dir(
 		const char *dir,
@@ -940,11 +995,8 @@ verify_dir(
 		size_t skippath,
 		const char *mfest)
 {
-	DIR *d;
-	struct dirent *e;
 	char **dentries = NULL;
 	size_t dentrieslen = 0;
-	size_t dentriessize = 0;
 	size_t curelem = 0;
 	size_t curdentry = 0;
 	char *entry;
@@ -985,38 +1037,16 @@ verify_dir(
 	 * easily flag missing entries in either list without hashing or
 	 * anything.
 	 */
-	if ((d = opendir(dir)) != NULL) {
-		while ((e = readdir(d)) != NULL) {
-			/* skip all dotfiles and Manifest files */
-			if (e->d_name[0] == '.' ||
-					strcmp(e->d_name, str_manifest) == 0 ||
-					strcmp(e->d_name, str_manifest_gz) == 0 ||
-					strcmp(e->d_name, str_manifest_files_gz) == 0)
+	if (list_dir(&dentries, &dentrieslen, dir) == 0) {
+		while (curdentry < dentrieslen) {
+			if (strcmp(dentries[curdentry], str_manifest) == 0 ||
+					strcmp(dentries[curdentry], str_manifest_gz) == 0 ||
+					strcmp(dentries[curdentry], str_manifest_files_gz) == 0)
 			{
+				curdentry++;
 				continue;
 			}
 
-			if (dentrieslen == dentriessize) {
-				dentriessize += LISTSZ;
-				dentries = realloc(dentries,
-						dentriessize * sizeof(dentries[0]));
-				if (dentries == NULL) {
-					fprintf(stderr, "out of memory\n");
-					return 1;
-				}
-			}
-			dentries[dentrieslen] = strdup(e->d_name);
-			if (dentries[dentrieslen] == NULL) {
-				fprintf(stderr, "out of memory\n");
-				return 1;
-			}
-			dentrieslen++;
-		}
-		closedir(d);
-
-		qsort(dentries, dentrieslen, sizeof(dentries[0]), compare_strings);
-
-		while (curdentry < dentrieslen) {
 			if (curelem < elemslen) {
 				entry = elems[curelem] + 2 + skippath;
 				etpe = *elems[curelem];

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     b5e2823a29eab0905cb78f77b0edbe954fb742b7
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 10 18:24:00 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Jul 10 18:24:00 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=b5e2823a

pts/rsync-generation/update-rsync-master: fix PYTHONPATH

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index b169a4a3f2..0cd30671bc 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -20,7 +20,7 @@ RSYNCDIR="${BASE_PATH}/master-rsync-tree"
 #### ---- Portage setup (use local modified copy) ---- ####
 
 PORTAGE_BASE_PATH="${BASE_PATH}/prefix/usr/lib/portage/"
-PYTHONPATH="${PORTAGE_BASE_PATH}/pym"
+PYTHONPATH="${PORTAGE_BASE_PATH}/lib"
 PORTAGE_CONFIGROOT="${BASE_PATH}/misc/config_root"
 PORTAGE_DEPCACHEDIR="${BASE_PATH}/depcache"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     990de73f5a0673988dc0644293d3a51452da1458
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 14 04:30:41 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jun 14 04:30:41 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=990de73f

scripts/rsync-generation/mksnapshot: don't download a crap 404 message

Fail instead of downloading a 404/403/500 whatever message.

Bug: https://bugs.gentoo.org/934308
Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 4d45ef2727..9f0a26abbf 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -22,7 +22,7 @@ BOOTSTRAP_SNAPSHOT=$( \
 	sed -n 's/^.*PV="\([0-9]\+\)"\s*$/portage-\1.tar.bz2/p' \
 )
 if [[ ! -s "${BOOTSTRAP_SNAPSHOT}" ]] ; then
-	curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
+	curl -f -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
 fi
 
 rm -Rf "${TMPDIR}"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f6acc1e57a57e74b0424973a5c79d07feeb74eee
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 10:24:42 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 10:24:42 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f6acc1e5

scripts/rsync-generation/mksnapshot: disable all but bz2 snapshots

who am I kidding, the bootstrap and the mirror retaining is all
hardwired to bz2, and other compression formats don't really make a big
enough dent to warrant lots of changes to allow picking a different
compresion format

disable all but bz2, that's the only thing we use afterall

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 5b8776b45e..4d45ef2727 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -44,10 +44,16 @@ popd > /dev/null || exit 1
 
 rm -Rf "${TMPDIR}"
 
+# The differences in size (57/52/47MB) for bz2,zstd,lz are not really that
+# big considering gzip's size (75MB) but the bootstrap-prefix script and
+# the logic above rely on .bz2 snapshot, so in reality no other format
+# than bzip2-compressed is necessary.  Since bzip2 is available
+# everywhere, or bootstrapped just fine for a long long time, stick with
+# it, for it is good enough for its purpose here
 COMPRS=(
 	"bz2:bzip2 -c -9"
-	"lz:lzip -c -9"
-	"zst:zstd -c -19"
+#	"lz:lzip -c -9"
+#	"zst:zstd -c -19"
 )
 
 # produce compressed variants, use as much cpu as left on the system, do

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     9edffa4de19c5906e02a23ff5149eac2b66fc38f
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 10:07:39 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 10:07:39 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=9edffa4d

scripts/rsync-generation/mksnapshot: xz can read lzip archives, so drop xz

lzip (lzma) compresses better than xz (lzma2) but xz can read/decompress
lzip archives, so drop the xz archive in favour of lzip

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 2fbdcfae0f..5b8776b45e 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -46,7 +46,6 @@ rm -Rf "${TMPDIR}"
 
 COMPRS=(
 	"bz2:bzip2 -c -9"
-	"xz:xz -c -9"
 	"lz:lzip -c -9"
 	"zst:zstd -c -19"
 )

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     29055666a0ad23b72e20081f40811b5202b445c7
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 08:51:35 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 08:51:35 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=29055666

scripts/rsync-generation/mksnapshot: drop gzip snapshot

gzip compresses to around 75MB which is much higher than bzip2's 57MB

The gzip snapshot was historically never used ever by the bootstrap
script, it always used and hardcoded bz2.  So don't bother creating a
gzip variant.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 66ad8a91ed..2fbdcfae0f 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -45,7 +45,6 @@ popd > /dev/null || exit 1
 rm -Rf "${TMPDIR}"
 
 COMPRS=(
-	"gz:gzip -c -9"
 	"bz2:bzip2 -c -9"
 	"xz:xz -c -9"
 	"lz:lzip -c -9"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     bd139cb929506a78bc3eab87e2e24a8e38b3032f
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 08:43:50 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 08:43:50 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=bd139cb9

scripts/rsync-generation/mksnapshot: increase zstd compression level

Using -9 we end up at 58MB, which is more than gzip2's 57MB.  So up the
level to -19 (the max, basically like how we configure the other
compressors too) to get 52MB, which is still much more than xz (48MB)
and lzip (47MB).

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 3accd67637..66ad8a91ed 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -49,7 +49,7 @@ COMPRS=(
 	"bz2:bzip2 -c -9"
 	"xz:xz -c -9"
 	"lz:lzip -c -9"
-	"zst:zstd -c -k -f -9"
+	"zst:zstd -c -19"
 )
 
 # produce compressed variants, use as much cpu as left on the system, do

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     34687be68301c57de465565393b2d3843a170e94
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 08:27:23 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 08:27:23 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=34687be6

scripts/rsync-generation/mksnapshot: fix zstd file extension

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 6fad8b895d..3accd67637 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -49,7 +49,7 @@ COMPRS=(
 	"bz2:bzip2 -c -9"
 	"xz:xz -c -9"
 	"lz:lzip -c -9"
-	"zstd:zstd -k -f -9"
+	"zst:zstd -c -k -f -9"
 )
 
 # produce compressed variants, use as much cpu as left on the system, do

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f61f449ad80c27e5ccaba5c8baa7ff1ee524b235
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 30 18:43:13 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 30 18:43:13 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f61f449a

scripts/rsync-generation/mksnapshot: also compress using lzip

lzip seems to reach a better compression ratio than xz, let's see if we
can use it

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index f932102daa..6fad8b895d 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -48,6 +48,7 @@ COMPRS=(
 	"gz:gzip -c -9"
 	"bz2:bzip2 -c -9"
 	"xz:xz -c -9"
+	"lz:lzip -c -9"
 	"zstd:zstd -k -f -9"
 )
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     9d09093d2d36c4bf651594f209aadc2e79da7a8e
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 30 11:15:09 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 30 11:15:09 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=9d09093d

scripts/rsync-generation/mksnapshot: start generating Zstd snapshots

Prepare for Zstandard becoming a standard, replacing another compressor.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 55 ++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 22 deletions(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 22808d58d4..1feece5577 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -44,20 +44,29 @@ popd > /dev/null || exit 1
 
 rm -Rf "${TMPDIR}"
 
-# be nice
-nice -n19 bzip2 -c -9 "${SNAME}" > "${SNAME}".bz2 &
-nice -n19 xz    -c -9 "${SNAME}" > "${SNAME}".xz  &
-nice -n19 gzip  -c -9 "${SNAME}" > "${SNAME}".gz  &
+COMPRS=(
+	"gz:gzip -c -9"
+	"bz2:bzip2 -c -9"
+	"xz:xz -c -9"
+	"zstd:zstd -k -f -9"
+)
+
+# produce compressed variants, use as much cpu as left on the system, do
+# all in parallel
+for compr in "${COMPRS[@]}" ; do
+	read -a args <<< "${compr#*:}"
+	nice -n19 "${args[@]}" "${SNAME}" > "${SNAME}.${compr%%:*}" &
+done
 wait
 
 # generate accompanying meta files
-md5sum "${SNAME##*/}"      > "${SNAME}".xz.umd5sum
-md5sum "${SNAME##*/}".xz   > "${SNAME}".xz.md5sum
-md5sum "${SNAME##*/}"      > "${SNAME}".bz2.umd5sum
-md5sum "${SNAME##*/}".bz2  > "${SNAME}".bz2.md5sum
-md5sum "${SNAME##*/}"      > "${SNAME}".gz.umd5sum
-md5sum "${SNAME##*/}".bz2  > "${SNAME}".gz.md5sum
-# use passphrase-fd to pass password
+for compr in "${COMPRS[@]}" ; do
+	compr=${compr%%:*}
+	md5sum "${SNAME##*/}"          > "${SNAME}.${compr}.umd5sum"
+	md5sum "${SNAME##*/}.${compr}" > "${SNAME}.${compr}.md5sum"
+done
+
+# create GPG detached signature, use passphrase-fd to pass password
 gpgopts=(
 	"--quiet"
 	"--batch"
@@ -68,18 +77,20 @@ gpgopts=(
 	"--detach-sign"
 	"--armor"
 )
-gpg "${gpgopts[@]}" -o "${SNAME}".xz.gpgsig  "${SNAME}".xz  \
-	< "${SCRIPTLOC}"/autosigner.pwd
-gpg "${gpgopts[@]}" -o "${SNAME}".bz2.gpgsig "${SNAME}".bz2 \
-	< "${SCRIPTLOC}"/autosigner.pwd
-gpg "${gpgopts[@]}" -o "${SNAME}".gz.gpgsig  "${SNAME}".gz  \
-	< "${SCRIPTLOC}"/autosigner.pwd
-
-# we no longer need the tar
+for compr in "${COMPRS[@]}" ; do
+	compr=${compr%%:*}
+	gpg "${gpgopts[@]}" -o "${SNAME}.${compr}.gpgsig" "${SNAME}.${compr}" \
+		< "${SCRIPTLOC}"/autosigner.pwd
+done
+
+# we no longer need the (original/uncompressed) tar
 rm "${SNAME}"
 
 # make convenience symlinks
-for f in {xz,bz2,gz}{,.gpgsig,.md5sum,.umd5sum} ; do
-	rm "portage-latest.tar.$f"
-	ln -s "${SNAME##*/}.$f" "portage-latest.tar.$f"
+for compr in "${COMPRS[@]}" ; do
+	compr=${compr%%:*}
+	for f in "${compr}"{,.gpgsig,.md5sum,.umd5sum} ; do
+		rm "portage-latest.tar.${f}"
+		ln -s "${SNAME##*/}.${f}" "portage-latest.tar.${f}"
+	done
 done

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     ed6eb030f4bf489a935891eb5fbdd1f5b0dd659d
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 28 18:48:36 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 28 18:48:36 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=ed6eb030

scripts/rsync-generation/mksnapshot: shellcheck

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 7879adc353..22808d58d4 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -14,7 +14,7 @@ SNAME=${PWD}/portage-$(date +%Y%m%d -d @${YESTERDAY}).tar
 TMPDIR=${PWD}/tmp-prefix-snapshot
 
 # clean up
-find . -maxdepth 2 -daystart -ctime +4 -type f | xargs --no-run-if-empty rm
+find . -maxdepth 2 -daystart -ctime +4 -type f -exec rm '{}' +
 
 # pull in active snapshot
 BOOTSTRAP_SNAPSHOT=$( \
@@ -31,7 +31,7 @@ mkdir -p "${TMPDIR}"
 # quickly take a snapshot, such that we get a consistent image
 pushd "${RSYNCTREE}" > /dev/null || exit 1
 tar -cf "${SNAME}" --exclude=snapshots -- * || exit 1
-popd > /dev/null
+popd > /dev/null || exit 1
 
 # now revamp it such that it's in a directory "portage"
 rm -Rf "${TMPDIR}"
@@ -40,7 +40,7 @@ pushd "${TMPDIR}" > /dev/null || exit 1
 mkdir portage
 tar -xf "${SNAME}" -C portage/
 tar --numeric-owner --format=posix --hard-dereference -cf "${SNAME}" portage/
-popd > /dev/null
+popd > /dev/null || exit 1
 
 rm -Rf "${TMPDIR}"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     3cbbb77fd201b19b99a31d35b7be7b91e9b3dd36
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 29 10:30:11 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Mar 29 10:30:11 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=3cbbb77f

scripts/rsync-generation/update-rsync-master: disable shellcheck fps

Disable false positives, unfortunately including the most helpful check
SC2086.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 304877ac95..b169a4a3f2 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+#shellcheck disable=SC2016,SC2086
+#SC2016: expressions don't expand in single quotes -> purposely in sed
+#SC2086: double quote to prevent word splitting -> exactly what we need w/ set
 
 SCRIPTSTARTTIME=$(date +%s)
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     e766d7ea7dae4c0f454a3e3a69f2f6bb3aba5ec8
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 28 14:56:42 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 28 14:56:42 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=e766d7ea

scripts/rsync-generation/push-rsync1: update to current usage

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/push-rsync1.sh | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/scripts/rsync-generation/push-rsync1.sh b/scripts/rsync-generation/push-rsync1.sh
index fc15894ecd..c9df9a379d 100755
--- a/scripts/rsync-generation/push-rsync1.sh
+++ b/scripts/rsync-generation/push-rsync1.sh
@@ -1,25 +1,23 @@
 #!/usr/bin/env bash
 
-TRGDIR="rsync1.domain::rsync-push_module"
+TRGDIR="/path/to/rsync0-prefix-tree"
 LOCALDIR="/path/to/master-rsync-tree"
 
-# figure out what's currently the active rsync1 mirror
-#active=$(ssh $TRGHOST "readlink $TRGDIR/rsync1")
-#echo "($(date +"%F %R")) current active snapshot on rsync1: ${active}"
-
-#case $active in
-#	rsync1a)  target=rsync1b ;;
-#	rsync1b)  target=rsync1a ;;
-#	*)
-#		echo "don't know what the active rsync1 mirror is: '$active'" > /dev/stderr
-#		exit 1
-#	;;
-#esac
-echo "($(date +"%F %R")) will refresh and activate snapshot: ${target}"
+echo "($(date +"%F %R")) will refresh snapshot"
 
 # synchronise the target
-rsync -va --delete "${LOCALDIR}"/ ${TRGDIR}/ > /var/tmp/rsync-updates.log || exit 1
+rsync -vca --delete --exclude=snapshots/ \
+	"${LOCALDIR}"/ ${TRGDIR}/ > /var/tmp/rsync-updates.log || exit 1
 
+PUBLICDIR=
+case $(hostname) in
+    disabled)
+        PUBLICDIR="somehost::gentoo-portage-prefix-push"
+        ;;
+esac
 # switch the active rsync1 mirror
-echo "($(date +"%F %R")) rsync done, switching target now"
-#ssh $TRGHOST "cd ${TRGDIR} && rm rsync1 && ln -s ${target} rsync1"
+if [[ -n ${PUBLICDIR} ]] ; then
+    rsync -vca --delete "${TRGDIR}"/ "${PUBLICDIR}"/
+fi
+
+echo "($(date +"%F %R")) rsync done"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     d4ce7ca1b7141ecebc27a660f8a34fcb4f9bec35
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 28 15:42:07 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 28 15:42:07 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=d4ce7ca1

scripts/rsync-generation/update-rsync-master: shellcheck

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 34 ++++++++++++++-----------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 6a93207518..304877ac95 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -3,7 +3,7 @@
 SCRIPTSTARTTIME=$(date +%s)
 
 # get keys for ssh and signing
-eval $(env SHELL=/bin/bash keychain -q --noask --eval)
+eval "$(env SHELL=/bin/bash keychain -q --noask --eval)"
 
 BASE_PATH="$(readlink -f "${BASH_SOURCE[0]%/*}")"
 
@@ -20,7 +20,6 @@ PORTAGE_BASE_PATH="${BASE_PATH}/prefix/usr/lib/portage/"
 PYTHONPATH="${PORTAGE_BASE_PATH}/pym"
 PORTAGE_CONFIGROOT="${BASE_PATH}/misc/config_root"
 PORTAGE_DEPCACHEDIR="${BASE_PATH}/depcache"
-MANIFEST_CACHE="${BASE_PATH}/manifests"
 
 # for .cvsps and gnupg cache mainly
 HOME="${BASE_PATH}/misc"
@@ -34,7 +33,12 @@ echo "(init) PATH=$PATH"
 
 #### ---- egencache settings ---- ####
 
-EGENCACHE_OPTS="--jobs=$(nproc) --load-average=$(nproc) --tolerant --update-use-local-desc"
+EGENCACHE_OPTS=(
+	"--jobs=$(nproc)"
+	"--load-average=$(nproc)"
+	"--tolerant"
+	"--update-use-local-desc"
+)
 
 export PYTHONPATH PORTDIR PORTAGE_BASE_PATH PORTAGE_CONFIGROOT  \
 	ROOT PORTAGE_TMPFS FEATURES HOME
@@ -66,8 +70,8 @@ apply_git_mtimes() {
 		git log --pretty=%ct --name-status --reverse "${from}..${to}"
 		echo 999  # end marker to trigger the last block to be done
 	} | \
-	while read line ; do
-		case ${line} in
+	while read -r line ; do
+		case "${line}" in
 			[0-9][0-9][0-9]*)
 				if [[ ${ts} -gt 0 ]] ; then
 					[[ ${#files[@]} == 0 ]] || \
@@ -78,11 +82,11 @@ apply_git_mtimes() {
 				;;
 			[ACMT]*)
 				set -- ${line}
-				files+=( $2 )
+				files+=( "$2" )
 				;;
 			[R]*)
 				set -- ${line}
-				files+=( $3 )
+				files+=( "$3" )
 				;;
 			[D]*)
 				set -- ${line}
@@ -95,8 +99,8 @@ apply_git_mtimes() {
 					# if the entire package was removed, touch the
 					# category level metadata
 					[[ -f ${f%/*}/metadata.xml ]] \
-						&& files+=( ${f%/*}/metadata.xml ) \
-						|| files+=( ${f%/*/*}/metadata.xml )
+						&& files+=( "${f%/*}"/metadata.xml ) \
+						|| files+=( "${f%/*/*}"/metadata.xml )
 				fi
 				;;
 		esac
@@ -201,8 +205,8 @@ echo "($(date +"%F %R")) git image updated"
 echo "($(date +"%F %R")) rsync Prefix tree to rsync-master"
 for entry in scripts *-*/* ; do
 	# copy it over
-	[[ -e ${RSYNCDIR}/${entry} ]] || mkdir -p "${RSYNCDIR}"/${entry}
-	rsync -v --delete -aC "${PREFIXTREEDIR}"/${entry}/ "${RSYNCDIR}"/${entry}/
+	[[ -e ${RSYNCDIR}/${entry} ]] || mkdir -p "${RSYNCDIR}/${entry}"
+	rsync -v --delete -aC "${PREFIXTREEDIR}/${entry}"/ "${RSYNCDIR}/${entry}"/
 done
 
 # we excluded the eclasses above, because we "overlay" them from gx86
@@ -230,7 +234,7 @@ START=$(date +%s)
 # generate the metadata
 echo "($(date +"%F %R")) generating metadata"
 dolog() {
-	echo $*
+	echo "$*"
 	"$@"
 }
 dolog "${PORTAGE_BASE_PATH}/bin/egencache" --update --rsync \
@@ -247,7 +251,7 @@ sync-type = rsync
 sync-uri = rsync://dont-sync
 auto-sync = no
 ' \
-	${EGENCACHE_OPTS} \
+	"${EGENCACHE_OPTS[@]}" \
 	|| exit 5
 
 STOP=$(date +%s)
@@ -281,8 +285,8 @@ sed -e '/^thin-manifests/s/true/false/' \
 # Signing is done with our snapshot signing key, and only on the top
 # level Manifest, for it covers indirectly the entire tree
 # remember, HOME is set to misc/ so .gnupg keychain lives there
-cat "${BASE_PATH}"/autosigner.pwd | \
-	qmanifest -g -p -s "0xC6317B3C" "${RSYNCDIR}" || \
+qmanifest -g -p -s "0xC6317B3C" "${RSYNCDIR}" \
+	< "${BASE_PATH}"/autosigner.pwd || \
 	echo "Manifest generation and/or signing failed!" >> /dev/stderr
 
 echo "($(date +"%F %R")) Manifest signed"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f9a6be590436b9fd5e59c8edcca75767f60a9c9d
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 28 15:11:30 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 28 15:11:30 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f9a6be59

scripts/rsync-generation/mksnapshot: shellcheck

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 88 +++++++++++++++-------------------
 1 file changed, 39 insertions(+), 49 deletions(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index e4db893b64..7879adc353 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -25,71 +25,61 @@ if [[ ! -s "${BOOTSTRAP_SNAPSHOT}" ]] ; then
 	curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
 fi
 
-rm -Rf ${TMPDIR}
-mkdir -p ${TMPDIR}
+rm -Rf "${TMPDIR}"
+mkdir -p "${TMPDIR}"
 
 # quickly take a snapshot, such that we get a consistent image
-pushd ${RSYNCTREE} > /dev/null
-tar -cf ${SNAME} --exclude=snapshots * || exit 1
+pushd "${RSYNCTREE}" > /dev/null || exit 1
+tar -cf "${SNAME}" --exclude=snapshots -- * || exit 1
 popd > /dev/null
 
 # now revamp it such that it's in a directory "portage"
-rm -Rf ${TMPDIR}
-mkdir -p ${TMPDIR}
-pushd ${TMPDIR} > /dev/null
+rm -Rf "${TMPDIR}"
+mkdir -p "${TMPDIR}"
+pushd "${TMPDIR}" > /dev/null || exit 1
 mkdir portage
-tar -xf ${SNAME} -C portage/
-tar --numeric-owner --format=posix --hard-dereference -cf ${SNAME} portage/
+tar -xf "${SNAME}" -C portage/
+tar --numeric-owner --format=posix --hard-dereference -cf "${SNAME}" portage/
 popd > /dev/null
 
-rm -Rf ${TMPDIR}
+rm -Rf "${TMPDIR}"
 
 # be nice
-nice -n19 bzip2 -c -9 ${SNAME} > ${SNAME}.bz2 &
-nice -n19 xz -c -9 ${SNAME} > ${SNAME}.xz &
-nice -n19 gzip -c -9 ${SNAME} > ${SNAME}.gz &
+nice -n19 bzip2 -c -9 "${SNAME}" > "${SNAME}".bz2 &
+nice -n19 xz    -c -9 "${SNAME}" > "${SNAME}".xz  &
+nice -n19 gzip  -c -9 "${SNAME}" > "${SNAME}".gz  &
 wait
 
 # generate accompanying meta files
-md5sum ${SNAME##*/}      > ${SNAME}.xz.umd5sum
-md5sum ${SNAME##*/}.xz   > ${SNAME}.xz.md5sum
-md5sum ${SNAME##*/}      > ${SNAME}.bz2.umd5sum
-md5sum ${SNAME##*/}.bz2  > ${SNAME}.bz2.md5sum
-md5sum ${SNAME##*/}      > ${SNAME}.gz.umd5sum
-md5sum ${SNAME##*/}.bz2  > ${SNAME}.gz.md5sum
-# gpg is really stupid, or I am too stupid to find the right option
-gpgopts="--quiet --batch --no-tty --passphrase-fd 0 --pinentry-mode loopback"
-gpgopts+=" --default-key C6317B3C --detach-sign --armor"
-gpg ${gpgopts} -o ${SNAME}.xz.gpgsig ${SNAME}.xz < ${SCRIPTLOC}/autosigner.pwd
-gpg ${gpgopts} -o ${SNAME}.bz2.gpgsig ${SNAME}.bz2 < ${SCRIPTLOC}/autosigner.pwd
-gpg ${gpgopts} -o ${SNAME}.gz.gpgsig ${SNAME}.gz < ${SCRIPTLOC}/autosigner.pwd
+md5sum "${SNAME##*/}"      > "${SNAME}".xz.umd5sum
+md5sum "${SNAME##*/}".xz   > "${SNAME}".xz.md5sum
+md5sum "${SNAME##*/}"      > "${SNAME}".bz2.umd5sum
+md5sum "${SNAME##*/}".bz2  > "${SNAME}".bz2.md5sum
+md5sum "${SNAME##*/}"      > "${SNAME}".gz.umd5sum
+md5sum "${SNAME##*/}".bz2  > "${SNAME}".gz.md5sum
+# use passphrase-fd to pass password
+gpgopts=(
+	"--quiet"
+	"--batch"
+	"--no-tty"
+	"--passphrase-fd" 0
+	"--pinentry-mode" "loopback"
+	"--default-key" "C6317B3C"
+	"--detach-sign"
+	"--armor"
+)
+gpg "${gpgopts[@]}" -o "${SNAME}".xz.gpgsig  "${SNAME}".xz  \
+	< "${SCRIPTLOC}"/autosigner.pwd
+gpg "${gpgopts[@]}" -o "${SNAME}".bz2.gpgsig "${SNAME}".bz2 \
+	< "${SCRIPTLOC}"/autosigner.pwd
+gpg "${gpgopts[@]}" -o "${SNAME}".gz.gpgsig  "${SNAME}".gz  \
+	< "${SCRIPTLOC}"/autosigner.pwd
 
 # we no longer need the tar
-rm ${SNAME}
+rm "${SNAME}"
 
 # make convenience symlinks
 for f in {xz,bz2,gz}{,.gpgsig,.md5sum,.umd5sum} ; do
-	rm portage-latest.tar.$f
-	ln -s ${SNAME##*/}.$f portage-latest.tar.$f
+	rm "portage-latest.tar.$f"
+	ln -s "${SNAME##*/}.$f" "portage-latest.tar.$f"
 done
-
-# darkside's delta code
-
-# FAILS and nobody cares!
-
-#YESTERDAY=$(date +%Y%m%d -d @${YESTERDAY})
-#TODAY=$(date +%Y%m%d -d @${TODAY})
-#cp portage-{${YESTERDAY},${TODAY}}.tar.bz2 /dev/shm/
-#SNAP_DIR=${PWD}
-#
-#cd /dev/shm
-#bunzip2 portage*
-#
-#differ -f bdelta portage-{${YESTERDAY},${TODAY}}.tar \
-#    ${SNAP_DIR}/deltas/snapshot-${YESTERDAY}-${TODAY}.patch
-#
-#bzip2 "${SNAP_DIR}/deltas/snapshot-${YESTERDAY}-${TODAY}.patch"
-#
-#rm -f portage* snapshot*
-
-# FAILS and nobody cares

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     6f38e3bd7e84b8d7c8df5229b4a5f1a34cb4843a
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 28 15:25:18 2024 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 28 15:25:18 2024 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=6f38e3bd

scripts/rsync-generation/refresh-mirror: shellcheck

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/refresh-mirror.sh | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index 6e2baffaa4..cbcb677266 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -1,4 +1,8 @@
 #!/bin/bash -l
+#shellcheck disable=SC2009,SC2030,SC2031
+#SC2009: consider using pgrep instead of grepping ps output
+#SC2030: modification of FD is local (to subshell)
+#SC2031: FD was modified in subshell
 
 # invocation script meant to be launched from cron
 
@@ -10,7 +14,7 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 	# allow one run to be skipped quietly
 	if [[ $((laststart + (40 * 60))) -lt ${now} ]] ; then
 		echo "another rsync-master generation process is still busy"
-		type pstree > /dev/null && pstree -A -l -p $(head -n1 ${LOGFILE})
+		type pstree > /dev/null && pstree -A -l -p "$(head -n1 ${LOGFILE})"
 		ps -ef | grep '[r]efresh-mirror'
 		tail ${LOGFILE}
 	else
@@ -22,16 +26,16 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 		pid=$(head -n1 ${LOGFILE})
 		if [[ ${pid} -gt 0 ]] ; then
 			echo "Killing stray/stuck processes"
-			pstree -A -l -c -p ${pid} | grep -o '[0-9]\+' | xargs kill
+			pstree -A -l -c -p "${pid}" | grep -o '[0-9]\+' | xargs kill
 			rm /tmp/rsync-master-busy
 		fi
 	fi
 else
-	mv ${LOGFILE} ${LOGFILE%.log}-prev.log
-	cd "$(readlink -f "${BASH_SOURCE[0]%/*}")"
+	mv "${LOGFILE}" "${LOGFILE%.log}"-prev.log
+	cd "$(readlink -f "${BASH_SOURCE[0]%/*}")" || exit 1
 	touch /tmp/rsync-master-busy
-	echo $$ > ${LOGFILE}
-	echo "starting generation $(date)" >> ${LOGFILE}
+	echo $$ > "${LOGFILE}"
+	echo "starting generation $(date)" >> "${LOGFILE}"
 	genandpush() {
 		./update-rsync-master.sh \
 			&& ./push-rsync1.sh

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     10296d4bb3af8fd716065b2cdf3423c1274d075a
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 11 10:37:44 2023 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Sep 11 10:37:44 2023 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=10296d4b

scripts/rsync-generation/mksnapshot: only *add* snapshots

don't overwrite existing snapshots, only add so as to avoid changing
generated snapshots

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/mksnapshot.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index d53d135693..e4db893b64 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -21,7 +21,9 @@ BOOTSTRAP_SNAPSHOT=$( \
 	grep -A1 MKSNAPSHOT-ANCHOR "${RSYNCTREE}"/scripts/bootstrap-prefix.sh | \
 	sed -n 's/^.*PV="\([0-9]\+\)"\s*$/portage-\1.tar.bz2/p' \
 )
-curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
+if [[ ! -s "${BOOTSTRAP_SNAPSHOT}" ]] ; then
+	curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
+fi
 
 rm -Rf ${TMPDIR}
 mkdir -p ${TMPDIR}

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Benda XU]

commit:     395b646f1154d1c1c625387d8decb3a1b8bed254
Author:     Benda Xu <heroxbd <AT> gentoo <DOT> org>
AuthorDate: Tue Aug  1 02:39:51 2023 +0000
Commit:     Benda XU <heroxbd <AT> gentoo <DOT> org>
CommitDate: Tue Aug  1 02:39:51 2023 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=395b646f

s/r-g/update-rsync-master.sh: put an alias gentoo repo name.

Closes: https://bugs.gentoo.org/911543
Signed-off-by: Benda Xu <heroxbd <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index f4b12787d4..6a93207518 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -271,8 +271,11 @@ START=$(date +%s)
 
 echo "($(date +"%F %R")) signing Manifest"
 
-# we will generate thick manifests, so ensure Portage knows that
-sed -i -e '/^thin-manifests/s/true/false/' "${RSYNCDIR}"/metadata/layout.conf
+# we will generate thick manifests, so ensure Portage knows that.
+# add a "gentoo" alias for compatibility, bug #911543.
+sed -e '/^thin-manifests/s/true/false/' \
+    -e '$arepo-name = gentoo_prefix\naliases = gentoo' \
+    -i "${RSYNCDIR}"/metadata/layout.conf
 
 # generate Thick Manifests
 # Signing is done with our snapshot signing key, and only on the top

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     6c70b47a0c9a8961b1115b486bda44d3d22aa42b
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Apr  9 16:05:26 2023 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Apr  9 16:06:15 2023 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=6c70b47a

scripts/rsync-generation/update-rsync-master.sh: use stronger git reset

use git reset --hard HEAD to bring the tree out of any conflict or
pollution state

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 26 +++++++++++--------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index d2e72ffec6..f4b12787d4 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -41,6 +41,12 @@ export PYTHONPATH PORTDIR PORTAGE_BASE_PATH PORTAGE_CONFIGROOT  \
 
 #### ---- git mtime helper ---- ####
 
+update_git_tree() {
+	git reset -q --hard HEAD
+	git clean -dfq
+	git pull -q
+}
+
 apply_git_mtimes() {
 	local from=$1
 	local to=$2
@@ -106,9 +112,7 @@ GLOBALSTART=${START}
 echo "($(date +"%F %R")) updating DTDs"
 pushd "$DTDDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git checkout -q .
-git clean -dfq
-git pull -q
+update_git_tree
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
 popd || exit 1
@@ -121,9 +125,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/dtd/timestamp.ch
 echo "($(date +"%F %R")) updating GLSAs"
 pushd "$GLSADIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git checkout -q .
-git clean -dfq
-git pull -q
+update_git_tree
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
 popd || exit 1
@@ -136,9 +138,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/glsa/timestamp.c
 echo "($(date +"%F %R")) updating news"
 pushd "$NEWSDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git checkout -q .
-git clean -dfq
-git pull -q
+update_git_tree
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
 popd || exit 1
@@ -165,9 +165,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating the gx86 tree"
 pushd "${GENTOOX86DIR}" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git checkout -q .
-git clean -dfq
-git pull -q
+update_git_tree
 tocommit=$(git log --pretty=format:'%H' -n1)
 gx86tscommit=$(git log --pretty=format:'%H %ct %cI' -n1 "${tocommit}")
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -193,9 +191,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating Prefix tree (Git image)"
 pushd "$PREFIXTREEDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git checkout -q .
-git clean -dfq
-git pull -q
+update_git_tree
 tocommit=$(git log --pretty=format:'%H' -n1)
 pfxtscommit=$(git log --pretty=format:'%H %ct %cI' -n1 "${tocommit}")
 apply_git_mtimes "${fromcommit}" "${tocommit}"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     3cd8bfde8bf2f6f408706bab10600ef96443d91b
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 17 19:26:40 2022 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Aug 17 19:26:40 2022 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=3cd8bfde

scripts/rsync-generation/update-rsync-master: zap any changes before pull

Somehow the git trees can get dirty, ensure they are completely clean
each time.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index d345da831e..d2e72ffec6 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -106,6 +106,7 @@ GLOBALSTART=${START}
 echo "($(date +"%F %R")) updating DTDs"
 pushd "$DTDDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git checkout -q .
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
@@ -120,6 +121,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/dtd/timestamp.ch
 echo "($(date +"%F %R")) updating GLSAs"
 pushd "$GLSADIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git checkout -q .
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
@@ -134,6 +136,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/glsa/timestamp.c
 echo "($(date +"%F %R")) updating news"
 pushd "$NEWSDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git checkout -q .
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
@@ -162,6 +165,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating the gx86 tree"
 pushd "${GENTOOX86DIR}" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git checkout -q .
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
@@ -189,6 +193,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating Prefix tree (Git image)"
 pushd "$PREFIXTREEDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git checkout -q .
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     80079ed04901570727c71c6f027bf2cc59276ab0
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 24 20:10:54 2022 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Jul 24 20:10:54 2022 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=80079ed0

scripts/rsync-generation/update-rsync-master: blast if you can

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index a8da6c894b..d345da831e 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -34,7 +34,7 @@ echo "(init) PATH=$PATH"
 
 #### ---- egencache settings ---- ####
 
-EGENCACHE_OPTS="--jobs=4 --load-average=3 --tolerant --update-use-local-desc"
+EGENCACHE_OPTS="--jobs=$(nproc) --load-average=$(nproc) --tolerant --update-use-local-desc"
 
 export PYTHONPATH PORTDIR PORTAGE_BASE_PATH PORTAGE_CONFIGROOT  \
 	ROOT PORTAGE_TMPFS FEATURES HOME

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     d9320d1986c9ba3b9b168e5fbc14b72899c67f6b
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Jun  7 05:14:56 2019 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jun  7 05:14:56 2019 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=d9320d19

scripts/rsync-generation: update README

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/README.new-portage-usage | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/rsync-generation/README.new-portage-usage b/scripts/rsync-generation/README.new-portage-usage
index 6f9511bb55..8229e519c5 100644
--- a/scripts/rsync-generation/README.new-portage-usage
+++ b/scripts/rsync-generation/README.new-portage-usage
@@ -1,9 +1,5 @@
-- fix shebang of bin/egencache
-- fix pym/portage/const_autotool.py
-- fix bin/ebuild.sh (PORTAGE_BASE)
-
-- set prefix symlink to point to the portage sources dir
-
+- unpack the sources of the portage to use
+- run (adapt prefix/offset-prefix arguments):
 configure \
 	--prefix="/home/prefix/rsync-master/scripts/prefix/usr" \
 	--with-offset-prefix="/home/prefix/rsync-master/scripts/prefix" \

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     9020aa9bd3b6dc20703eec64817b8978197120b4
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Jun  7 05:41:40 2019 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jun  7 05:41:40 2019 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=9020aa9b

scripts/rsync-generation/update-rsync-master: hashgen -> qmanifest

Now hashgen is integrated in portage-utils as qmanifest, switch to using
it.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/rsync-generation/update-rsync-master.sh | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index d5afe16c2e..a8da6c894b 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -274,23 +274,12 @@ echo "($(date +"%F %R")) signing Manifest"
 sed -i -e '/^thin-manifests/s/true/false/' "${RSYNCDIR}"/metadata/layout.conf
 
 # generate Thick Manifests
-${BASE_PATH}/hashgen "${RSYNCDIR}"
-
 # Signing is done with our snapshot signing key, and only on the top
 # level Manifest, for it covers indirectly the entire tree
-
 # remember, HOME is set to misc/ so .gnupg keychain lives there
-gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
-	--pinentry-mode loopback \
-	--sign --clearsign --digest-algo SHA512 \
-	--yes "${RSYNCDIR}"/Manifest \
-	< "${BASE_PATH}"/autosigner.pwd 2>&1
-if [[ -f ${RSYNCDIR}/Manifest.asc ]] ; then
-	touch -r "${RSYNCDIR}"/Manifest "${RSYNCDIR}"/Manifest.asc
-	mv "${RSYNCDIR}"/Manifest{.asc,}
-else
-	echo "signing failed!" >> /dev/stderr
-fi
+cat "${BASE_PATH}"/autosigner.pwd | \
+	qmanifest -g -p -s "0xC6317B3C" "${RSYNCDIR}" || \
+	echo "Manifest generation and/or signing failed!" >> /dev/stderr
 
 echo "($(date +"%F %R")) Manifest signed"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     3befa7a80d74502d7d635a33856949efd7c815ac
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon May 14 15:51:00 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon May 14 15:51:00 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=3befa7a8

scripts/rsync-generation/update-rsync-master: include timestamp.commit files

 scripts/rsync-generation/update-rsync-master.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 859eb9df55..d5afe16c2e 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -165,6 +165,7 @@ fromcommit=$(git log --pretty=format:'%H' -n1)
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
+gx86tscommit=$(git log --pretty=format:'%H %ct %cI' -n1 "${tocommit}")
 apply_git_mtimes "${fromcommit}" "${tocommit}"
 popd || exit 1
 rsync -v \
@@ -191,6 +192,7 @@ fromcommit=$(git log --pretty=format:'%H' -n1)
 git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
+pfxtscommit=$(git log --pretty=format:'%H %ct %cI' -n1 "${tocommit}")
 apply_git_mtimes "${fromcommit}" "${tocommit}"
 echo "($(date +"%F %R")) git image updated"
 
@@ -253,6 +255,8 @@ TIME_EGENCACHE=$((STOP - START))
 date -u > "${RSYNCDIR}"/metadata/timestamp
 date -u '+%s %c %Z' > "${RSYNCDIR}"/metadata/timestamp.x
 date -R -u > "${RSYNCDIR}"/metadata/timestamp.chk
+echo "${gx86tscommit}" > "${RSYNCDIR}"/metadata/timestamp.commit
+echo "${pfxtscommit}" > "${RSYNCDIR}"/metadata/timestamp.commit.prefix-tree
 echo "($(date +"%F %R")) set date to $(<"${RSYNCDIR}"/metadata/timestamp.chk)"
 
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     21d5c5b399b2645f33ff25faaeb1219e3c5c9b52
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 29 05:54:58 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar 29 05:54:58 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=21d5c5b3

scripts/rsync-generation/update-rsync-master: deal with nested files (filesdir) in apply_git_mtimes

 scripts/rsync-generation/update-rsync-master.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 4eac483b88..859eb9df55 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -83,9 +83,14 @@ apply_git_mtimes() {
 				# in case a file is removed, ensure Manifest gets
 				# updated by touching a file which should be there
 				if [[ $2 == */*/* ]] ; then
-					[[ -f ${2%/*}/metadata.xml ]] \
-						&& files+=( ${2%/*}/metadata.xml ) \
-						|| files+=( ${2%/*/*}/metadata.xml )
+					local f=${2}
+					# treat anything in files at the package level
+					[[ ${f} == */files/* ]] && f=${f%/files/*}/foo
+					# if the entire package was removed, touch the
+					# category level metadata
+					[[ -f ${f%/*}/metadata.xml ]] \
+						&& files+=( ${f%/*}/metadata.xml ) \
+						|| files+=( ${f%/*/*}/metadata.xml )
 				fi
 				;;
 		esac

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     711ddc91052b7664f9f620bfa00992c5a15f48a2
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 27 14:03:23 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Tue Mar 27 14:03:23 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=711ddc91

scripts/rsync-generation/update-rsync-master: touch cat metadata.xml upon package removal

 scripts/rsync-generation/update-rsync-master.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 02807d19c9..4eac483b88 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -79,10 +79,14 @@ apply_git_mtimes() {
 				files+=( $3 )
 				;;
 			[D]*)
+				set -- ${line}
 				# in case a file is removed, ensure Manifest gets
 				# updated by touching a file which should be there
-				[[ $2 == */* ]] && \
-					files+=( ${2%/*}/metadata.xml )
+				if [[ $2 == */*/* ]] ; then
+					[[ -f ${2%/*}/metadata.xml ]] \
+						&& files+=( ${2%/*}/metadata.xml ) \
+						|| files+=( ${2%/*/*}/metadata.xml )
+				fi
 				;;
 		esac
 	done

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     9e0ef57926049281e6075a8040c6c99871878ca5
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 17 20:58:11 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 17 20:58:11 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=9e0ef579

scripts/rsync-generation/hashgen: moved to its own repository

hashgen will live on in github.com/grobian/hashgen

 scripts/rsync-generation/hashgen.c | 1510 ------------------------------------
 1 file changed, 1510 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
deleted file mode 100644
index 60121978da..0000000000
--- a/scripts/rsync-generation/hashgen.c
+++ /dev/null
@@ -1,1510 +0,0 @@
-/* Copyright 2006-2018 Gentoo Foundation; Distributed under the GPL v2 */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <ctype.h>
-#include <dirent.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/time.h>
-#include <openssl/sha.h>
-#include <openssl/whrlpool.h>
-#include <blake2.h>
-#include <zlib.h>
-#include <gpgme.h>
-
-/* Generate thick Manifests based on thin Manifests */
-
-/* In order to build this program, the following packages are required:
- * - app-crypt/libb2 (for BLAKE2, for as long as openssl doesn't include it)
- * - dev-libs/openssl (for SHA, WHIRLPOOL)
- * - sys-libs/zlib (for compressing Manifest files)
- * - app-crypt/gpgme (for signing/verifying the top level manifest)
- * compile like this:
- *   ${CC} -o hashgen -fopenmp ${CFLAGS} \
- *         -lssl -lcrypto -lb2 -lz `gpgme-config --libs` hashgen.c
- */
-
-enum hash_impls {
-	HASH_SHA256    = 1<<0,
-	HASH_SHA512    = 1<<1,
-	HASH_WHIRLPOOL = 1<<2,
-	HASH_BLAKE2B   = 1<<3
-};
-/* default changed from sha256, sha512, whirlpool
- * to blake2b, sha512 on 2017-11-21 */
-#define HASH_DEFAULT  (HASH_BLAKE2B | HASH_SHA512);
-static int hashes = HASH_DEFAULT;
-
-static inline void
-hex_hash(char *out, const unsigned char *buf, const int length)
-{
-	switch (length) {
-		/* SHA256_DIGEST_LENGTH */
-		case 32:
-			snprintf(out, 64 + 1,
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x",
-					buf[ 0], buf[ 1], buf[ 2], buf[ 3], buf[ 4],
-					buf[ 5], buf[ 6], buf[ 7], buf[ 8], buf[ 9],
-					buf[10], buf[11], buf[12], buf[13], buf[14],
-					buf[15], buf[16], buf[17], buf[18], buf[19],
-					buf[20], buf[21], buf[22], buf[23], buf[24],
-					buf[25], buf[26], buf[27], buf[28], buf[29],
-					buf[30], buf[31]
-					);
-			break;
-		/* SHA512_DIGEST_LENGTH, WHIRLPOOL_DIGEST_LENGTH, BLAKE2B_OUTBYTES */
-		case 64:
-			snprintf(out, 128 + 1,
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-					"%02x%02x%02x%02x",
-					buf[ 0], buf[ 1], buf[ 2], buf[ 3], buf[ 4],
-					buf[ 5], buf[ 6], buf[ 7], buf[ 8], buf[ 9],
-					buf[10], buf[11], buf[12], buf[13], buf[14],
-					buf[15], buf[16], buf[17], buf[18], buf[19],
-					buf[20], buf[21], buf[22], buf[23], buf[24],
-					buf[25], buf[26], buf[27], buf[28], buf[29],
-					buf[30], buf[31], buf[32], buf[33], buf[34],
-					buf[35], buf[36], buf[37], buf[38], buf[39],
-					buf[40], buf[41], buf[42], buf[43], buf[44],
-					buf[45], buf[46], buf[47], buf[48], buf[49],
-					buf[50], buf[51], buf[52], buf[53], buf[54],
-					buf[55], buf[56], buf[57], buf[58], buf[59],
-					buf[60], buf[61], buf[62], buf[63]
-					);
-			break;
-		/* fallback case, should never be necessary */
-		default:
-			{
-				int i;
-				for (i = 0; i < length; i++) {
-					snprintf(&out[i * 2], 3, "%02x", buf[i]);
-				}
-			}
-			break;
-	}
-}
-
-static inline void
-update_times(struct timeval *tv, struct stat *s)
-{
-#ifdef __MACH__
-# define st_mtim st_mtimespec
-# define st_atim st_atimespec
-#endif
-	if (tv[1].tv_sec < s->st_mtim.tv_sec ||
-			(tv[1].tv_sec == s->st_mtim.tv_sec &&
-			 tv[1].tv_usec < s->st_mtim.tv_nsec / 1000))
-	{
-		tv[0].tv_sec = s->st_atim.tv_sec;
-		tv[0].tv_usec = s->st_atim.tv_nsec / 1000;
-		tv[1].tv_sec = s->st_mtim.tv_sec;
-		tv[1].tv_usec = s->st_mtim.tv_nsec / 1000;
-	}
-}
-
-static void
-get_hashes(
-		const char *fname,
-		char *sha256,
-		char *sha512,
-		char *whrlpl,
-		char *blak2b,
-		size_t *flen)
-{
-	FILE *f;
-	char data[8192];
-	size_t len;
-	SHA256_CTX s256;
-	SHA512_CTX s512;
-	WHIRLPOOL_CTX whrl;
-	blake2b_state bl2b;
-
-	if ((f = fopen(fname, "r")) == NULL)
-		return;
-
-	SHA256_Init(&s256);
-	SHA512_Init(&s512);
-	WHIRLPOOL_Init(&whrl);
-	blake2b_init(&bl2b, BLAKE2B_OUTBYTES);
-
-	while ((len = fread(data, 1, sizeof(data), f)) > 0) {
-		*flen += len;
-#pragma omp parallel sections
-		{
-#pragma omp section
-			{
-				if (hashes & HASH_SHA256)
-					SHA256_Update(&s256, data, len);
-			}
-#pragma omp section
-			{
-				if (hashes & HASH_SHA512)
-					SHA512_Update(&s512, data, len);
-			}
-#pragma omp section
-			{
-				if (hashes & HASH_WHIRLPOOL)
-					WHIRLPOOL_Update(&whrl, data, len);
-			}
-#pragma omp section
-			{
-				if (hashes & HASH_BLAKE2B)
-					blake2b_update(&bl2b, (unsigned char *)data, len);
-			}
-		}
-	}
-	fclose(f);
-
-#pragma omp parallel sections
-	{
-		{
-			if (hashes & HASH_SHA256) {
-				unsigned char sha256buf[SHA256_DIGEST_LENGTH];
-				SHA256_Final(sha256buf, &s256);
-				hex_hash(sha256, sha256buf, SHA256_DIGEST_LENGTH);
-			}
-		}
-#pragma omp section
-		{
-			if (hashes & HASH_SHA512) {
-				unsigned char sha512buf[SHA512_DIGEST_LENGTH];
-				SHA512_Final(sha512buf, &s512);
-				hex_hash(sha512, sha512buf, SHA512_DIGEST_LENGTH);
-			}
-		}
-#pragma omp section
-		{
-			if (hashes & HASH_WHIRLPOOL) {
-				unsigned char whrlplbuf[WHIRLPOOL_DIGEST_LENGTH];
-				WHIRLPOOL_Final(whrlplbuf, &whrl);
-				hex_hash(whrlpl, whrlplbuf, WHIRLPOOL_DIGEST_LENGTH);
-			}
-		}
-#pragma omp section
-		{
-			if (hashes & HASH_BLAKE2B) {
-				unsigned char blak2bbuf[BLAKE2B_OUTBYTES];
-				blake2b_final(&bl2b, blak2bbuf, BLAKE2B_OUTBYTES);
-				hex_hash(blak2b, blak2bbuf, BLAKE2B_OUTBYTES);
-			}
-		}
-	}
-}
-
-#define LISTSZ 64
-
-static int
-compare_strings(const void *l, const void *r)
-{
-	const char **strl = (const char **)l;
-	const char **strr = (const char **)r;
-	return strcmp(*strl, *strr);
-}
-
-/**
- * Return a sorted list of entries in the given directory.  All entries
- * starting with a dot are ignored, and not present in the returned
- * list.  The list and all entries are allocated using malloc() and need
- * to be freed.
- * This function returns 0 when everything is fine, non-zero otherwise.
- */
-static char
-list_dir(char ***retlist, size_t *retcnt, const char *path)
-{
-	DIR *d;
-	struct dirent *e;
-	size_t rlen = 0;
-	size_t rsize = 0;
-	char **rlist = NULL;
-
-	if ((d = opendir(path)) != NULL) {
-		while ((e = readdir(d)) != NULL) {
-			/* skip all dotfiles */
-			if (e->d_name[0] == '.')
-				continue;
-
-			if (rlen == rsize) {
-				rsize += LISTSZ;
-				rlist = realloc(rlist,
-						rsize * sizeof(rlist[0]));
-				if (rlist == NULL) {
-					fprintf(stderr, "out of memory\n");
-					return 1;
-				}
-			}
-			rlist[rlen] = strdup(e->d_name);
-			if (rlist[rlen] == NULL) {
-				fprintf(stderr, "out of memory\n");
-				return 1;
-			}
-			rlen++;
-		}
-		closedir(d);
-
-		qsort(rlist, rlen, sizeof(rlist[0]), compare_strings);
-
-		*retlist = rlist;
-		*retcnt = rlen;
-		return 0;
-	} else {
-		return 1;
-	}
-}
-
-static void
-write_hashes(
-		struct timeval *tv,
-		const char *root,
-		const char *name,
-		const char *type,
-		FILE *m,
-		gzFile gm)
-{
-	size_t flen = 0;
-	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
-	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
-	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
-	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
-	char data[8192];
-	char fname[8192];
-	size_t len;
-	struct stat s;
-
-	snprintf(fname, sizeof(fname), "%s/%s", root, name);
-
-	if (stat(fname, &s) != 0)
-		return;
-
-	update_times(tv, &s);
-
-	get_hashes(fname, sha256, sha512, whrlpl, blak2b, &flen);
-
-	len = snprintf(data, sizeof(data), "%s %s %zd", type, name, flen);
-	if (hashes & HASH_BLAKE2B)
-		len += snprintf(data + len, sizeof(data) - len,
-				" BLAKE2B %s", blak2b);
-	if (hashes & HASH_SHA256)
-		len += snprintf(data + len, sizeof(data) - len,
-				" SHA256 %s", sha256);
-	if (hashes & HASH_SHA512)
-		len += snprintf(data + len, sizeof(data) - len,
-				" SHA512 %s", sha512);
-	if (hashes & HASH_WHIRLPOOL)
-		len += snprintf(data + len, sizeof(data) - len,
-				" WHIRLPOOL %s", whrlpl);
-	len += snprintf(data + len, sizeof(data) - len, "\n");
-
-	if (m != NULL)
-		fwrite(data, len, 1, m);
-	if (gm != NULL)
-		gzwrite(gm, data, len);
-}
-
-static char
-write_hashes_dir(
-		struct timeval *tv,
-		const char *root,
-		const char *name,
-		gzFile zm)
-{
-	char path[8192];
-	char **dentries;
-	size_t dentrieslen;
-	size_t i;
-
-	snprintf(path, sizeof(path), "%s/%s", root, name);
-	if (list_dir(&dentries, &dentrieslen, path) == 0) {
-		for (i = 0; i < dentrieslen; i++) {
-			snprintf(path, sizeof(path), "%s/%s", name, dentries[i]);
-			free(dentries[i]);
-			if (write_hashes_dir(tv, root, path, zm) == 0)
-				continue;
-			/* regular file */
-			write_hashes(tv, root, path, "DATA", NULL, zm);
-		}
-		free(dentries);
-		return 0;
-	} else {
-		return 1;
-	}
-}
-
-static char
-process_files(struct timeval *tv, const char *dir, const char *off, FILE *m)
-{
-	char path[8192];
-	char **dentries;
-	size_t dentrieslen;
-	size_t i;
-
-	snprintf(path, sizeof(path), "%s/%s", dir, off);
-	if (list_dir(&dentries, &dentrieslen, path) == 0) {
-		for (i = 0; i < dentrieslen; i++) {
-			snprintf(path, sizeof(path), "%s%s%s",
-					off, *off == '\0' ? "" : "/", dentries[i]);
-			free(dentries[i]);
-			if (process_files(tv, dir, path, m) == 0)
-				continue;
-			/* regular file */
-			write_hashes(tv, dir, path, "AUX", m, NULL);
-		}
-		free(dentries);
-		return 0;
-	} else {
-		return 1;
-	}
-}
-
-static int
-parse_layout_conf(const char *path)
-{
-	FILE *f;
-	char buf[8192];
-	size_t len = 0;
-	size_t sz;
-	char *p;
-	char *q;
-	char *tok;
-	char *last_nl;
-	char *start;
-	int ret = 0;
-
-	if ((f = fopen(path, "r")) == NULL)
-		return 0;
-
-	/* read file, examine lines after encountering a newline, that is,
-	 * if the file doesn't end with a newline, the final bit is ignored */
-	while ((sz = fread(buf + len, 1, sizeof(buf) - len, f)) > 0) {
-		len += sz;
-		start = buf;
-		last_nl = NULL;
-		for (p = buf; p - buf < len; p++) {
-			if (*p == '\n') {
-				if (last_nl != NULL)
-					start = last_nl + 1;
-				last_nl = p;
-				do {
-					sz = strlen("manifest-hashes");
-					if (strncmp(start, "manifest-hashes", sz))
-						break;
-					if ((q = strchr(start + sz, '=')) == NULL)
-						break;
-					q++;
-					while (isspace((int)*q))
-						q++;
-					/* parse the tokens, whitespace separated */
-					tok = q;
-					do {
-						while (!isspace((int)*q))
-							q++;
-						sz = q - tok;
-						if (strncmp(tok, "SHA256", sz) == 0) {
-							ret |= HASH_SHA256;
-						} else if (strncmp(tok, "SHA512", sz) == 0) {
-							ret |= HASH_SHA512;
-						} else if (strncmp(tok, "WHIRLPOOL", sz) == 0) {
-							ret |= HASH_WHIRLPOOL;
-						} else if (strncmp(tok, "BLAKE2B", sz) == 0) {
-							ret |= HASH_BLAKE2B;
-						} else {
-							fprintf(stderr, "warning: unsupported hash from "
-									"layout.conf: %.*s\n", (int)sz, tok);
-						}
-						while (isspace((int)*q) && *q != '\n')
-							q++;
-						tok = q;
-					} while (*q != '\n');
-					/* got it, expect only once, so stop processing */
-					fclose(f);
-					return ret;
-				} while (0);
-			}
-		}
-		if (last_nl != NULL) {
-			last_nl++;  /* skip \n */
-			len = last_nl - buf;
-			memmove(buf, last_nl, len);
-			last_nl = buf;
-		} else {
-			/* skip too long line */
-			len = 0;
-		}
-	}
-
-	fclose(f);
-	/* if we didn't find anything, return the default set */
-	return HASH_DEFAULT;
-}
-
-static char *str_manifest = "Manifest";
-static char *str_manifest_gz = "Manifest.gz";
-static char *str_manifest_files_gz = "Manifest.files.gz";
-enum type_manifest {
-	GLOBAL_MANIFEST,   /* Manifest.files.gz + Manifest */
-	SUBTREE_MANIFEST,  /* Manifest.gz for recursive list of files */
-	EBUILD_MANIFEST,   /* Manifest thick from thin */
-	CATEGORY_MANIFEST  /* Manifest.gz with Manifest entries */
-};
-static char *
-generate_dir(const char *dir, enum type_manifest mtype)
-{
-	FILE *f;
-	char path[8192];
-	struct stat s;
-	struct timeval tv[2];
-	char **dentries;
-	size_t dentrieslen;
-	size_t i;
-
-	/* our timestamp strategy is as follows:
-	 * - when a Manifest exists, use its timestamp
-	 * - when a meta-Manifest is written (non-ebuilds) use the timestamp
-	 *   of the latest Manifest referenced
-	 * - when a Manifest is written for something like eclasses, use the
-	 *   timestamp of the latest file in the dir
-	 * this way we should keep updates limited to where changes are, and
-	 * also get reproducible mtimes. */
-	tv[0].tv_sec = 0;
-	tv[0].tv_usec = 0;
-	tv[1].tv_sec = 0;
-	tv[1].tv_usec = 0;
-
-	if (mtype == GLOBAL_MANIFEST) {
-		char *mfest;
-		size_t len;
-		gzFile mf;
-		time_t rtime;
-
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_files_gz);
-		if ((mf = gzopen(path, "wb9")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					path, strerror(errno));
-			return NULL;
-		}
-
-		/* These "IGNORE" entries are taken from gx86, there is no
-		 * standardisation on this, on purpose, apparently. */
-		len = snprintf(path, sizeof(path),
-				"IGNORE distfiles\n"
-				"IGNORE local\n"
-				"IGNORE lost+found\n"
-				"IGNORE packages\n");
-		gzwrite(mf, path, len);
-
-		if (list_dir(&dentries, &dentrieslen, dir) != 0)
-			return NULL;
-
-		for (i = 0; i < dentrieslen; i++) {
-			/* ignore existing Manifests */
-			if (strcmp(dentries[i], str_manifest_files_gz) == 0 ||
-					strcmp(dentries[i], str_manifest) == 0)
-			{
-				free(dentries[i]);
-				continue;
-			}
-
-			snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
-
-			mfest = NULL;
-			if (!stat(path, &s)) {
-				if (s.st_mode & S_IFDIR) {
-					if (
-							strcmp(dentries[i], "eclass")   == 0 ||
-							strcmp(dentries[i], "licenses") == 0 ||
-							strcmp(dentries[i], "metadata") == 0 ||
-							strcmp(dentries[i], "profiles") == 0 ||
-							strcmp(dentries[i], "scripts")  == 0
-					   )
-					{
-						mfest = generate_dir(path, SUBTREE_MANIFEST);
-					} else {
-						mfest = generate_dir(path, CATEGORY_MANIFEST);
-					}
-
-					if (mfest == NULL) {
-						fprintf(stderr, "generating Manifest for %s failed!\n",
-								path);
-						gzclose(mf);
-						return NULL;
-					}
-
-					snprintf(path, sizeof(path), "%s/%s",
-							dentries[i], mfest);
-					write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
-				} else if (s.st_mode & S_IFREG) {
-					write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
-				} /* ignore other "things" (like symlinks) as they
-					 don't belong in a tree */
-			} else {
-				fprintf(stderr, "stat(%s) failed: %s\n",
-						path, strerror(errno));
-			}
-			free(dentries[i]);
-		}
-		free(dentries);
-		gzclose(mf);
-
-		if (tv[0].tv_sec != 0) {
-			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_files_gz);
-			utimes(path, tv);
-		}
-
-		/* create global Manifest */
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
-		if ((f = fopen(path, "w")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					path, strerror(errno));
-			return NULL;
-		}
-
-		write_hashes(tv, dir, str_manifest_files_gz, "MANIFEST", f, NULL);
-		time(&rtime);
-		len = strftime(path, sizeof(path),
-				"TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n", gmtime(&rtime));
-		fwrite(path, len, 1, f);
-		fflush(f);
-		fclose(f);
-
-		/* because we write a timestamp in Manifest, we don't mess with
-		 * its mtime, else it would obviously lie */
-		return str_manifest_files_gz;
-	} else if (mtype == SUBTREE_MANIFEST) {
-		const char *ldir;
-		gzFile mf;
-
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
-		if ((mf = gzopen(path, "wb9")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					path, strerror(errno));
-			return NULL;
-		}
-
-		ldir = strrchr(dir, '/');
-		if (ldir == NULL)
-			ldir = dir;
-		if (strcmp(ldir, "metadata") == 0) {
-			size_t len;
-			len = snprintf(path, sizeof(path),
-					"IGNORE timestamp\n"
-					"IGNORE timestamp.chk\n"
-					"IGNORE timestamp.commit\n"
-					"IGNORE timestamp.x\n");
-			gzwrite(mf, path, len);
-		}
-
-		if (list_dir(&dentries, &dentrieslen, dir) != 0)
-			return NULL;
-
-		for (i = 0; i < dentrieslen; i++) {
-			/* ignore existing Manifests */
-			if (strcmp(dentries[i], str_manifest_gz) == 0) {
-				free(dentries[i]);
-				continue;
-			}
-
-			if (write_hashes_dir(tv, dir, dentries[i], mf) != 0)
-				write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
-			free(dentries[i]);
-		}
-
-		free(dentries);
-		gzclose(mf);
-
-		if (tv[0].tv_sec != 0) {
-			/* set Manifest and dir mtime to most recent file found */
-			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
-			utimes(path, tv);
-			utimes(dir, tv);
-		}
-
-		return str_manifest_gz;
-	} else if (mtype == CATEGORY_MANIFEST) {
-		char *mfest;
-		gzFile mf;
-
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
-		if ((mf = gzopen(path, "wb9")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					path, strerror(errno));
-			return NULL;
-		}
-
-		if (list_dir(&dentries, &dentrieslen, dir) != 0)
-			return NULL;
-
-		for (i = 0; i < dentrieslen; i++) {
-			/* ignore existing Manifests */
-			if (strcmp(dentries[i], str_manifest_gz) == 0) {
-				free(dentries[i]);
-				continue;
-			}
-
-			snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
-			if (!stat(path, &s)) {
-				if (s.st_mode & S_IFDIR) {
-					mfest = generate_dir(path, EBUILD_MANIFEST);
-
-					if (mfest == NULL) {
-						fprintf(stderr, "generating Manifest for %s failed!\n",
-								path);
-						gzclose(mf);
-						return NULL;
-					}
-
-					snprintf(path, sizeof(path), "%s/%s",
-							dentries[i], mfest);
-					write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
-				} else if (s.st_mode & S_IFREG) {
-					write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
-				} /* ignore other "things" (like symlinks) as they
-					 don't belong in a tree */
-			} else {
-				fprintf(stderr, "stat(%s) failed: %s\n",
-						path, strerror(errno));
-			}
-			free(dentries[i]);
-		}
-
-		free(dentries);
-		gzclose(mf);
-
-		if (tv[0].tv_sec != 0) {
-			/* set Manifest and dir mtime to most ebuild dir found */
-			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
-			utimes(path, tv);
-			utimes(dir, tv);
-		}
-
-		return str_manifest_gz;
-	} else if (mtype == EBUILD_MANIFEST) {
-		char newmanifest[8192];
-		FILE *m;
-
-		snprintf(newmanifest, sizeof(newmanifest), "%s/.Manifest.new", dir);
-		if ((m = fopen(newmanifest, "w")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					newmanifest, strerror(errno));
-			return NULL;
-		}
-
-		/* we know the Manifest is sorted, and stuff in files/ is
-		 * prefixed with AUX, hence, if it exists, we need to do it
-		 * first */
-		snprintf(path, sizeof(path), "%s/files", dir);
-		process_files(tv, path, "", m);
-
-		/* the Manifest file may be missing in case there are no DIST
-		 * entries to be stored */
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
-		if (!stat(path, &s))
-			update_times(tv, &s);
-		f = fopen(path, "r");
-		if (f != NULL) {
-			/* copy the DIST entries, we could do it unconditional, but this
-			 * way we can re-run without producing invalid Manifests */
-			while (fgets(path, sizeof(path), f) != NULL) {
-				if (strncmp(path, "DIST ", 5) == 0)
-					if (fwrite(path, strlen(path), 1, m) != 1) {
-						fprintf(stderr, "failed to write to "
-								"%s/.Manifest.new: %s\n",
-								dir, strerror(errno));
-						fclose(f);
-						fclose(m);
-						return NULL;
-					}
-			}
-			fclose(f);
-		}
-
-		if (list_dir(&dentries, &dentrieslen, dir) == 0) {
-			for (i = 0; i < dentrieslen; i++) {
-				if (strcmp(dentries[i] + strlen(dentries[i]) - 7,
-							".ebuild") != 0)
-				{
-					free(dentries[i]);
-					continue;
-				}
-				write_hashes(tv, dir, dentries[i], "EBUILD", m, NULL);
-				free(dentries[i]);
-			}
-			free(dentries);
-		}
-
-		write_hashes(tv, dir, "ChangeLog", "MISC", m, NULL);
-		write_hashes(tv, dir, "metadata.xml", "MISC", m, NULL);
-
-		fflush(m);
-		fclose(m);
-
-		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
-		rename(newmanifest, path);
-
-		if (tv[0].tv_sec != 0) {
-			/* set Manifest and dir mtime to most recent file we found */
-			utimes(path, tv);
-			utimes(dir, tv);
-		}
-
-		return str_manifest;
-	} else {
-		return NULL;
-	}
-}
-
-static char *
-process_dir_gen(const char *dir)
-{
-	char path[8192];
-	int newhashes;
-
-	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
-	if ((newhashes = parse_layout_conf(path)) != 0) {
-		hashes = newhashes;
-	} else {
-		return "generation must be done on a full tree";
-	}
-
-	if (chdir(dir) != 0) {
-		fprintf(stderr, "cannot chdir() to %s: %s\n", dir, strerror(errno));
-		return "not a directory";
-	}
-
-	if (generate_dir(".\0", GLOBAL_MANIFEST) == NULL)
-		return "generation failed";
-
-	return NULL;
-}
-
-static char
-verify_gpg_sig(const char *path)
-{
-	gpgme_ctx_t g_ctx;
-	gpgme_data_t manifest;
-	gpgme_data_t out;
-	gpgme_verify_result_t vres;
-	gpgme_signature_t sig;
-	gpgme_key_t key;
-	char buf[32];
-	FILE *f;
-	struct tm *ctime;
-	char ret = 1;  /* fail */
-
-	if ((f = fopen(path, "r")) == NULL) {
-		fprintf(stderr, "failed to open %s: %s\n", path, strerror(errno));
-		return ret;
-	}
-
-	if (gpgme_new(&g_ctx) != GPG_ERR_NO_ERROR) {
-		fprintf(stderr, "failed to create gpgme context\n");
-		return ret;
-	}
-
-	if (gpgme_data_new(&out) != GPG_ERR_NO_ERROR) {
-		fprintf(stderr, "failed to create new gpgme data\n");
-		return ret;
-	}
-
-	if (gpgme_data_new_from_stream(&manifest, f) != GPG_ERR_NO_ERROR) {
-		fprintf(stderr, "failed to create new gpgme data from stream\n");
-		return ret;
-	}
-
-	if (gpgme_op_verify(g_ctx, manifest, NULL, out) != GPG_ERR_NO_ERROR) {
-		fprintf(stderr, "failed to verify signature\n");
-		return ret;
-	}
-
-	vres = gpgme_op_verify_result(g_ctx);
-	fclose(f);
-
-	if (vres == NULL || vres->signatures == NULL) {
-		fprintf(stderr, "verification failed due to a missing gpg keyring\n");
-		return ret;
-	}
-
-	for (sig = vres->signatures; sig != NULL; sig = sig->next) {
-		if (sig->status != GPG_ERR_NO_PUBKEY) {
-			ctime = gmtime((time_t *)&sig->timestamp);
-			strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", ctime);
-			printf("%s key fingerprint "
-					"%.4s %.4s %.4s %.4s %.4s  %.4s %.4s %.4s %.4s %.4s\n"
-					"%s signature made %s by\n",
-					gpgme_pubkey_algo_name(sig->pubkey_algo),
-					sig->fpr +  0, sig->fpr +  4, sig->fpr +  8, sig->fpr + 12,
-					sig->fpr + 16, sig->fpr + 20, sig->fpr + 24, sig->fpr + 28,
-					sig->fpr + 32, sig->fpr + 36,
-					sig->status == GPG_ERR_NO_ERROR ? "good" : "BAD",
-					buf);
-
-			if (gpgme_get_key(g_ctx, sig->fpr, &key, 0) == GPG_ERR_NO_ERROR) {
-				ret = 0;  /* valid */
-				if (key->uids != NULL)
-					printf("%s\n", key->uids->uid);
-				gpgme_key_release(key);
-			}
-		}
-
-		switch (sig->status) {
-			case GPG_ERR_NO_ERROR:
-				/* nothing, handled above */
-				break;
-			case GPG_ERR_SIG_EXPIRED:
-				printf("the signature is valid but expired\n");
-				break;
-			case GPG_ERR_KEY_EXPIRED:
-				printf("the signature is valid but the key used to verify "
-						"the signature has expired\n");
-				break;
-			case GPG_ERR_CERT_REVOKED:
-				printf("the signature is valid but the key used to verify "
-						"the signature has been revoked\n");
-				break;
-			case GPG_ERR_BAD_SIGNATURE:
-				printf("the signature is invalid\n");
-				break;
-			case GPG_ERR_NO_PUBKEY:
-				printf("the signature could not be verified due to a "
-						"missing key\n");
-				break;
-			default:
-				printf("there was some other error which prevented the "
-						"signature verification\n");
-				break;
-		}
-	}
-
-	gpgme_release(g_ctx);
-
-	return ret;
-}
-
-static size_t checked_manifests = 0;
-static size_t checked_files = 0;
-static size_t failed_files = 0;
-
-static char
-verify_file(const char *dir, char *mfline, const char *mfest)
-{
-	char *path;
-	char *size;
-	long long int fsize;
-	char *hashtype;
-	char *hash;
-	char *p;
-	char buf[8192];
-	size_t flen = 0;
-	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
-	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
-	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
-	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
-	char ret = 0;
-
-	/* mfline is a Manifest file line with type and leading path
-	 * stripped, something like:
-	 * file <SIZE> <HASHTYPE HASH ...>
-	 * we parse this, and verify the size and hashes */
-
-	path = mfline;
-	p = strchr(path, ' ');
-	if (p == NULL) {
-		fprintf(stderr, "%s: corrupt manifest line: %s\n", mfest, path);
-		return 1;
-	}
-	*p++ = '\0';
-
-	size = p;
-	p = strchr(size, ' ');
-	if (p == NULL) {
-		fprintf(stderr, "%s: corrupt manifest line, need size for %s\n",
-				mfest, path);
-		return 1;
-	}
-	*p++ = '\0';
-	fsize = strtoll(size, NULL, 10);
-	if (fsize == 0 && errno == EINVAL) {
-		fprintf(stderr, "%s: corrupt manifest line, size is not a number: %s\n",
-				dir + 2, size);
-		return 1;
-	}
-
-	sha256[0] = sha512[0] = whrlpl[0] = blak2b[0] = '\0';
-	snprintf(buf, sizeof(buf), "%s/%s", dir, path);
-	get_hashes(buf, sha256, sha512, whrlpl, blak2b, &flen);
-
-	if (flen == 0) {
-		fprintf(stderr, "cannot locate %s/%s\n", dir + 2, path);
-		return 1;
-	}
-
-	checked_files++;
-
-	if (flen != fsize) {
-		printf("%s:%s:\n- file size mismatch\n"
-				"       got: %zd\n"
-				"  expected: %lld\n",
-				mfest, path, flen, fsize);
-		failed_files++;
-		return 1;
-	}
-
-	/* now we are in free territory, we read TYPE HASH pairs until we
-	 * drained the string, and match them against what we computed */
-	while (p != NULL && *p != '\0') {
-		hashtype = p;
-		p = strchr(hashtype, ' ');
-		if (p == NULL) {
-			fprintf(stderr, "%s: corrupt manifest line, missing hash type\n",
-					mfest);
-			return 1;
-		}
-		*p++ = '\0';
-
-		hash = p;
-		p = strchr(hash, ' ');
-		if (p != NULL)
-			*p++ = '\0';
-
-#define idif(X) if (X == 0) printf("%s:%s:\n", mfest, path);
-		if (strcmp(hashtype, "SHA256") == 0) {
-			if (!(hashes & HASH_SHA256)) {
-				idif(ret);
-				printf("- warning: hash SHA256 ignored as "
-						"it is not enabled for this repository\n");
-			} else if (strcmp(hash, sha256) != 0) {
-				idif(ret);
-				printf("- SHA256 hash mismatch\n"
-						"              computed: '%s'\n"
-						"  recorded in manifest: '%s'\n",
-						sha256, hash);
-				ret = 1;
-			}
-			sha256[0] = '\0';
-		} else if (strcmp(hashtype, "SHA512") == 0) {
-			if (!(hashes & HASH_SHA512)) {
-				idif(ret);
-				printf("- warning: hash SHA512 ignored as "
-						"it is not enabled for this repository\n");
-			} else if (strcmp(hash, sha512) != 0) {
-				idif(ret);
-				printf("- SHA512 hash mismatch\n"
-						"              computed: '%s'\n"
-						"  recorded in manifest: '%s'\n",
-						sha512, hash);
-				ret = 1;
-			}
-			sha512[0] = '\0';
-		} else if (strcmp(hashtype, "WHIRLPOOL") == 0) {
-			if (!(hashes & HASH_WHIRLPOOL)) {
-				idif(ret);
-				printf("- warning: hash WHIRLPOOL ignored as "
-						"it is not enabled for this repository\n");
-			} else if (strcmp(hash, whrlpl) != 0) {
-				idif(ret);
-				printf("- WHIRLPOOL hash mismatch\n"
-						"              computed: '%s'\n"
-						"  recorded in manifest: '%s'\n",
-						whrlpl, hash);
-				ret = 1;
-			}
-			whrlpl[0] = '\0';
-		} else if (strcmp(hashtype, "BLAKE2B") == 0) {
-			if (!(hashes & HASH_BLAKE2B)) {
-				idif(ret);
-				printf("- warning: hash BLAKE2B ignored as "
-						"it is not enabled for this repository\n");
-			} else if (strcmp(hash, blak2b) != 0) {
-				idif(ret);
-				printf("- BLAKE2B hash mismatch\n"
-						"              computed: '%s'\n"
-						"  recorded in manifest: '%s'\n",
-						blak2b, hash);
-				ret = 1;
-			}
-			blak2b[0] = '\0';
-		} else {
-			idif(ret);
-			printf("- unsupported hash: %s\n", hashtype);
-			ret = 1;
-		}
-	}
-
-	if (sha256[0] != '\0') {
-		idif(ret);
-		printf("- missing hash: SHA256\n");
-		ret = 1;
-	}
-	if (sha512[0] != '\0') {
-		idif(ret);
-		printf("- missing hash: SHA512\n");
-		ret = 1;
-	}
-	if (whrlpl[0] != '\0') {
-		idif(ret);
-		printf("- missing hash: WHIRLPOOL\n");
-		ret = 1;
-	}
-	if (blak2b[0] != '\0') {
-		idif(ret);
-		printf("- missing hash: BLAKE2B\n");
-		ret = 1;
-	}
-
-	failed_files += ret;
-	return ret;
-}
-
-static int
-compare_elems(const void *l, const void *r)
-{
-	const char *strl = *((const char **)l) + 2;
-	const char *strr = *((const char **)r) + 2;
-	unsigned char cl;
-	unsigned char cr;
-	/* compare treating / as end of string */
-	while ((cl = *strl++) == (cr = *strr++))
-		if (cl == '\0')
-			return 0;
-	if (cl == '/')
-		cl = '\0';
-	if (cr == '/')
-		cr = '\0';
-	return cl - cr;
-}
-
-struct subdir_workload {
-	size_t subdirlen;
-	size_t elemslen;
-	char **elems;
-};
-
-static char verify_manifest(const char *dir, const char *manifest);
-
-static char
-verify_dir(
-		const char *dir,
-		char **elems,
-		size_t elemslen,
-		size_t skippath,
-		const char *mfest)
-{
-	char **dentries = NULL;
-	size_t dentrieslen = 0;
-	size_t curelem = 0;
-	size_t curdentry = 0;
-	char *entry;
-	char *slash;
-	char etpe;
-	char ret = 0;
-	int cmp;
-	struct subdir_workload **subdir = NULL;
-	size_t subdirsize = 0;
-	size_t subdirlen = 0;
-
-	/* shortcut a single Manifest entry pointing to the same dir
-	 * (happens at top-level) */
-	if (elemslen == 1 && skippath == 0 &&
-			**elems == 'M' && strchr(*elems + 2, '/') == NULL)
-	{
-		if ((ret = verify_file(dir, *elems + 2, mfest)) == 0) {
-			slash = strchr(*elems + 2, ' ');
-			if (slash != NULL)
-				*slash = '\0';
-			/* else, verify_manifest will fail, so ret will be handled */
-			ret = verify_manifest(dir, *elems + 2);
-		}
-		return ret;
-	}
-
-	/*
-	 * We have a list of entries from the manifest just read, now we
-	 * need to match these onto the directory layout.  From what we got
-	 * - we can ignore TIMESTAMP and DIST entries
-	 * - IGNOREs need to be handled separate (shortcut)
-	 * - MANIFESTs need to be handled on their own, for memory
-	 *   consumption reasons, we defer them to until we've verified
-	 *   what's left, we treat the path the Manifest refers to as IGNORE
-	 * - DATAs, EBUILDs and MISCs needs verifying
-	 * - AUXs need verifying, but in files/ subdir
-	 * If we sort both lists, we should be able to do a merge-join, to
-	 * easily flag missing entries in either list without hashing or
-	 * anything.
-	 */
-	if (list_dir(&dentries, &dentrieslen, dir) == 0) {
-		while (curdentry < dentrieslen) {
-			if (strcmp(dentries[curdentry], str_manifest) == 0 ||
-					strcmp(dentries[curdentry], str_manifest_gz) == 0 ||
-					strcmp(dentries[curdentry], str_manifest_files_gz) == 0)
-			{
-				curdentry++;
-				continue;
-			}
-
-			if (curelem < elemslen) {
-				entry = elems[curelem] + 2 + skippath;
-				etpe = *elems[curelem];
-			} else {
-				entry = "";
-				etpe = 'I';
-			}
-
-			/* handle subdirs first */
-			if ((slash = strchr(entry, '/')) != NULL) {
-				size_t sublen = slash - entry;
-				int elemstart = curelem;
-				char **subelems = &elems[curelem];
-
-				/* collect all entries like this one (same subdir) into
-				 * a sub-list that we can verify */
-				curelem++;
-				while (curelem < elemslen &&
-						strncmp(entry, elems[curelem] + 2 + skippath,
-							sublen + 1) == 0)
-					curelem++;
-
-				if (subdirlen == subdirsize) {
-					subdirsize += LISTSZ;
-					subdir = realloc(subdir,
-							subdirsize * sizeof(subdir[0]));
-					if (subdir == NULL) {
-						fprintf(stderr, "out of memory\n");
-						return 1;
-					}
-				}
-				subdir[subdirlen] = malloc(sizeof(struct subdir_workload));
-				if (subdir[subdirlen] == NULL) {
-					fprintf(stderr, "out of memory\n");
-					return 1;
-				}
-				subdir[subdirlen]->subdirlen = sublen;
-				subdir[subdirlen]->elemslen = curelem - elemstart;
-				subdir[subdirlen]->elems = subelems;
-				subdirlen++;
-
-				curelem--; /* move back, see below */
-
-				/* modify the last entry to be the subdir, such that we
-				 * can let the code below synchronise with dentries */
-				elems[curelem][2 + skippath + sublen] = ' ';
-				entry = elems[curelem] + 2 + skippath;
-				etpe = 'S';  /* flag this was a subdir */
-			}
-
-			/* does this entry exist in list? */
-			if (*entry == '\0') {
-				/* end of list reached, force dir to catch up */
-				cmp = 1;
-			} else {
-				slash = strchr(entry, ' ');
-				if (slash != NULL)
-					*slash = '\0';
-				cmp = strcmp(entry, dentries[curdentry]);
-				if (slash != NULL)
-					*slash = ' ';
-			}
-			if (cmp == 0) {
-				/* equal, so yay */
-				if (etpe == 'D') {
-					ret |= verify_file(dir, entry, mfest);
-				}
-				/* else this is I(GNORE) or S(ubdir), which means it is
-				 * ok in any way (M shouldn't happen) */
-				curelem++;
-				curdentry++;
-			} else if (cmp < 0) {
-				/* entry is missing from dir */
-				if (etpe == 'I') {
-					/* right, we can ignore this */
-				} else {
-					ret |= 1;
-					slash = strchr(entry, ' ');
-					if (slash != NULL)
-						*slash = '\0';
-					printf("%s:%s:\n- %s file not found\n",
-							mfest, entry, etpe == 'M' ? "MANIFEST" : "DATA");
-					if (slash != NULL)
-						*slash = ' ';
-					failed_files++;
-				}
-				curelem++;
-			} else if (cmp > 0) {
-				/* dir has extra element */
-				ret |= 1;
-				printf("%s:\n- found excess file: %s\n",
-						mfest, dentries[curdentry]);
-				curdentry++;
-				failed_files++;
-			}
-		}
-
-		while (dentrieslen-- > 0)
-			free(dentries[dentrieslen]);
-		free(dentries);
-
-#pragma omp parallel for shared(ret) private(entry, etpe, slash)
-		for (cmp = 0; cmp < subdirlen; cmp++) {
-			char ndir[8192];
-
-			entry = subdir[cmp]->elems[0] + 2 + skippath;
-			etpe = subdir[cmp]->elems[0][0];
-
-			/* restore original entry format */
-			subdir[cmp]->elems[subdir[cmp]->elemslen - 1]
-				[2 + skippath + subdir[cmp]->subdirlen] = '/';
-
-			if (etpe == 'M') {
-				size_t skiplen = strlen(dir) + 1 + subdir[cmp]->subdirlen;
-				/* sub-Manifest, we need to do a proper recurse */
-				slash = strrchr(entry, '/');  /* cannot be NULL */
-				snprintf(ndir, sizeof(ndir), "%s/%s", dir, entry);
-				ndir[skiplen] = '\0';
-				slash = strchr(ndir + skiplen + 1, ' ');
-				if (slash != NULL)  /* path should fit in ndir ... */
-					*slash = '\0';
-				if (verify_file(dir, entry, mfest) != 0 ||
-						verify_manifest(ndir, ndir + skiplen + 1) != 0)
-					ret |= 1;
-			} else {
-				snprintf(ndir, sizeof(ndir), "%s/%.*s", dir,
-						(int)subdir[cmp]->subdirlen, entry);
-				ret |= verify_dir(ndir, subdir[cmp]->elems,
-						subdir[cmp]->elemslen,
-						skippath + subdir[cmp]->subdirlen + 1, mfest);
-			}
-
-			free(subdir[cmp]);
-		}
-
-		if (subdir)
-			free(subdir);
-
-		return ret;
-	} else {
-		return 1;
-	}
-}
-
-static char
-verify_manifest(const char *dir, const char *manifest)
-{
-	char buf[8192];
-	FILE *f;
-	gzFile mf;
-	char ret = 0;
-
-	size_t elemssize = 0;
-	size_t elemslen = 0;
-	char **elems = NULL;
-#define append_list(STR) \
-	if (strncmp(STR, "TIMESTAMP ", 10) != 0 || strncmp(STR, "DIST ", 5) != 0) {\
-		char *endp = STR + strlen(STR) - 1;\
-		while (isspace(*endp))\
-			*endp-- = '\0';\
-		if (elemslen == elemssize) {\
-			elemssize += LISTSZ;\
-			elems = realloc(elems, elemssize * sizeof(elems[0]));\
-			if (elems == NULL) {\
-				fprintf(stderr, "out of memory\n");\
-				return 1;\
-			}\
-		}\
-		if (strncmp(STR, "IGNORE ", 7) == 0) {\
-			STR[5] = 'I';\
-			elems[elemslen] = strdup(STR + 5);\
-			if (elems[elemslen] == NULL) {\
-				fprintf(stderr, "out of memory\n");\
-				return 1;\
-			}\
-			elemslen++;\
-		} else if (strncmp(STR, "MANIFEST ", 9) == 0) {\
-			STR[7] = 'M';\
-			elems[elemslen] = strdup(STR + 7);\
-			if (elems[elemslen] == NULL) {\
-				fprintf(stderr, "out of memory\n");\
-				return 1;\
-			}\
-			elemslen++;\
-		} else if (strncmp(STR, "DATA ", 5) == 0 ||\
-				strncmp(STR, "MISC ", 5) == 0 ||\
-				strncmp(STR, "EBUILD ", 7) == 0)\
-		{\
-			if (*STR == 'E') {\
-				STR[5] = 'D';\
-				elems[elemslen] = strdup(STR + 5);\
-			} else {\
-				STR[3] = 'D';\
-				elems[elemslen] = strdup(STR + 3);\
-			}\
-			if (elems[elemslen] == NULL) {\
-				fprintf(stderr, "out of memory\n");\
-				return 1;\
-			}\
-			elemslen++;\
-		} else if (strncmp(STR, "AUX ", 4) == 0) {\
-			/* translate directly into what it is: DATA in files/ */\
-			size_t slen = strlen(STR + 2) + sizeof("files/");\
-			elems[elemslen] = malloc(slen);\
-			if (elems[elemslen] == NULL) {\
-				fprintf(stderr, "out of memory\n");\
-				return 1;\
-			}\
-			snprintf(elems[elemslen], slen, "D files/%s", STR + 4);\
-			elemslen++;\
-		}\
-	}
-
-	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
-	if (strcmp(manifest, str_manifest) == 0) {
-		if ((f = fopen(buf, "r")) == NULL) {
-			fprintf(stderr, "failed to open %s: %s\n",
-					buf, strerror(errno));
-			return 1;
-		}
-		while (fgets(buf, sizeof(buf), f) != NULL) {
-			append_list(buf);
-		}
-		fclose(f);
-	} else if (strcmp(manifest, str_manifest_files_gz) == 0 ||
-			strcmp(manifest, str_manifest_gz) == 0)
-	{
-		if ((mf = gzopen(buf, "rb9")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for reading: %s\n",
-					buf, strerror(errno));
-			return 1;
-		}
-		while (gzgets(mf, buf, sizeof(buf)) != NULL) {
-			append_list(buf);
-		}
-		gzclose(mf);
-	}
-
-	/* The idea:
-	 * - Manifest without MANIFEST entries, we need to scan the entire
-	 *   subtree
-	 * - Manifest with MANIFEST entries, assume they are just one level
-	 *   deeper, thus ignore that subdir, further like above
-	 * - Manifest at top-level, needs to be igored as it only points to
-	 *   the larger Manifest.files.gz
-	 */
-	qsort(elems, elemslen, sizeof(elems[0]), compare_elems);
-	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
-	ret = verify_dir(dir, elems, elemslen, 0, buf + 2);
-	checked_manifests++;
-
-	while (elemslen-- > 0)
-		free(elems[elemslen]);
-	free(elems);
-
-	return ret;
-}
-
-static char *
-process_dir_vrfy(const char *dir)
-{
-	char buf[8192];
-	int newhashes;
-	char *ret = NULL;
-	struct timeval startt;
-	struct timeval finisht;
-	double etime;
-
-	gettimeofday(&startt, NULL);
-
-	fprintf(stdout, "verifying %s...\n", dir);
-	snprintf(buf, sizeof(buf), "%s/metadata/layout.conf", dir);
-	if ((newhashes = parse_layout_conf(buf)) != 0) {
-		hashes = newhashes;
-	} else {
-		return "verification must be done on a full tree";
-	}
-
-	if (chdir(dir) != 0) {
-		fprintf(stderr, "cannot chdir() to %s: %s\n", dir, strerror(errno));
-		return "not a directory";
-	}
-
-	if (verify_gpg_sig(str_manifest) != 0)
-		ret = "gpg signature invalid";
-
-	/* verification goes like this:
-	 * - verify the signature of the top-level Manifest file (done
-	 *   above)
-	 * - read the contents of the Manifest file, and process the
-	 *   entries - verify them, check there are no files which shouldn't
-	 *   be there
-	 * - recurse into directories for which Manifest files are defined
-	 */
-	if (verify_manifest(".\0", str_manifest) != 0)
-		ret = "manifest verification failed";
-
-	gettimeofday(&finisht, NULL);
-
-	etime = ((double)((finisht.tv_sec - startt.tv_sec) * 1000000 +
-				finisht.tv_usec) - (double)startt.tv_usec) / 1000000.0;
-	printf("checked %zd Manifests, %zd files, %zd failures in %.02fs\n",
-			checked_manifests, checked_files, failed_files, etime);
-	return ret;
-}
-
-int
-main(int argc, char *argv[])
-{
-	char *prog;
-	char *(*runfunc)(const char *);
-	int arg = 1;
-	int ret = 0;
-	char *rsn;
-
-	if ((prog = strrchr(argv[0], '/')) == NULL) {
-		prog = argv[0];
-	} else {
-		prog++;
-	}
-
-	if (argc > 1) {
-		if (strcmp(argv[1], "hashverify") == 0 ||
-				strcmp(argv[1], "hashgen") == 0)
-		{
-			prog = argv[1];
-			arg = 2;
-		}
-	}
-
-	if (strcmp(prog, "hashverify") == 0) {
-		runfunc = &process_dir_vrfy;
-	} else {
-		/* default mode: hashgen */
-		runfunc = &process_dir_gen;
-	}
-
-	gpgme_check_version(NULL);
-
-	if (argc > 1) {
-		for (; arg < argc; arg++) {
-			rsn = runfunc(argv[arg]);
-			if (rsn != NULL) {
-				printf("%s\n", rsn);
-				ret |= 1;
-			}
-		}
-	} else {
-		rsn = runfunc(".");
-		if (rsn != NULL) {
-			printf("%s\n", rsn);
-			ret |= 1;
-		}
-	}
-
-	return ret;
-}

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     22a85e115b85a0400b7d5f8f4fc6c3d12ba17cb1
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 17 20:56:34 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 17 20:56:34 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=22a85e11

scripts/rsync-generation/update-rsync-master: update package metadata.xml

 scripts/rsync-generation/update-rsync-master.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 4c1febc2c9..02807d19c9 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -81,7 +81,8 @@ apply_git_mtimes() {
 			[D]*)
 				# in case a file is removed, ensure Manifest gets
 				# updated by touching a file which should be there
-				files+=( metadata.xml )
+				[[ $2 == */* ]] && \
+					files+=( ${2%/*}/metadata.xml )
 				;;
 		esac
 	done

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     6e4d403adaa5efd03ee731a05c016d7c6affe6c8
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 12 10:06:14 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Mar 12 10:06:14 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=6e4d403a

scripts/rsync-generation/hashgen: report no garbage when key is missing

 scripts/rsync-generation/hashgen.c | 41 ++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index c4955f65c7..60121978da 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -829,24 +829,31 @@ verify_gpg_sig(const char *path)
 	vres = gpgme_op_verify_result(g_ctx);
 	fclose(f);
 
+	if (vres == NULL || vres->signatures == NULL) {
+		fprintf(stderr, "verification failed due to a missing gpg keyring\n");
+		return ret;
+	}
+
 	for (sig = vres->signatures; sig != NULL; sig = sig->next) {
-		ctime = gmtime((time_t *)&sig->timestamp);
-		strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", ctime);
-		printf("%s key fingerprint "
-				"%.4s %.4s %.4s %.4s %.4s  %.4s %.4s %.4s %.4s %.4s\n"
-				"%s signature made %s by\n",
-				gpgme_pubkey_algo_name(sig->pubkey_algo),
-				sig->fpr +  0, sig->fpr +  4, sig->fpr +  8, sig->fpr + 12,
-				sig->fpr + 16, sig->fpr + 20, sig->fpr + 24, sig->fpr + 28,
-				sig->fpr + 32, sig->fpr + 36,
-				sig->status == GPG_ERR_NO_ERROR ? "good" : "BAD",
-				buf);
-
-		if (gpgme_get_key(g_ctx, sig->fpr, &key, 0) == GPG_ERR_NO_ERROR) {
-			ret = 0;  /* valid */
-			if (key->uids != NULL)
-				printf("%s\n", key->uids->uid);
-			gpgme_key_release(key);
+		if (sig->status != GPG_ERR_NO_PUBKEY) {
+			ctime = gmtime((time_t *)&sig->timestamp);
+			strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", ctime);
+			printf("%s key fingerprint "
+					"%.4s %.4s %.4s %.4s %.4s  %.4s %.4s %.4s %.4s %.4s\n"
+					"%s signature made %s by\n",
+					gpgme_pubkey_algo_name(sig->pubkey_algo),
+					sig->fpr +  0, sig->fpr +  4, sig->fpr +  8, sig->fpr + 12,
+					sig->fpr + 16, sig->fpr + 20, sig->fpr + 24, sig->fpr + 28,
+					sig->fpr + 32, sig->fpr + 36,
+					sig->status == GPG_ERR_NO_ERROR ? "good" : "BAD",
+					buf);
+
+			if (gpgme_get_key(g_ctx, sig->fpr, &key, 0) == GPG_ERR_NO_ERROR) {
+				ret = 0;  /* valid */
+				if (key->uids != NULL)
+					printf("%s\n", key->uids->uid);
+				gpgme_key_release(key);
+			}
 		}
 
 		switch (sig->status) {

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     c85ac432678c09bcdc136237243b865561ff380f
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 10 15:04:42 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 10 15:04:42 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=c85ac432

scripts/rsync-generation/update-rsync-master: ensure deletes bump mtime

 scripts/rsync-generation/update-rsync-master.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 86a9a4773b..4c1febc2c9 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -78,6 +78,11 @@ apply_git_mtimes() {
 				set -- ${line}
 				files+=( $3 )
 				;;
+			[D]*)
+				# in case a file is removed, ensure Manifest gets
+				# updated by touching a file which should be there
+				files+=( metadata.xml )
+				;;
 		esac
 	done
 }

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     2ea68e88531288c29c5533440d63da6c8136d477
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Mar  7 18:03:38 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Mar  7 18:03:38 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=2ea68e88

scripts/rsync-generation/hashgen: count missing and stray files as failed

 scripts/rsync-generation/hashgen.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 7bedc75ca4..c4955f65c7 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1228,6 +1228,7 @@ verify_dir(
 							mfest, entry, etpe == 'M' ? "MANIFEST" : "DATA");
 					if (slash != NULL)
 						*slash = ' ';
+					failed_files++;
 				}
 				curelem++;
 			} else if (cmp > 0) {
@@ -1236,6 +1237,7 @@ verify_dir(
 				printf("%s:\n- found excess file: %s\n",
 						mfest, dentries[curdentry]);
 				curdentry++;
+				failed_files++;
 			}
 		}
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     4d65987d63d5c2e573ae2be35dc5427950ff3677
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar  3 21:42:43 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar  3 21:42:43 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=4d65987d

scripts/rsync-generation/hashgen: rewrite hashgen to be simpler

a logic rewrite was necessary to be able to properly generate the right
type of Manifest for the level at hand

 scripts/rsync-generation/hashgen.c | 424 ++++++++++++++++++++++---------------
 1 file changed, 256 insertions(+), 168 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 446299f92e..7bedc75ca4 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -342,13 +342,12 @@ write_hashes_dir(
 }
 
 static char
-process_files(const char *dir, const char *off, FILE *m)
+process_files(struct timeval *tv, const char *dir, const char *off, FILE *m)
 {
 	char path[8192];
 	char **dentries;
 	size_t dentrieslen;
 	size_t i;
-	struct timeval tv[2]; /* dummy, won't use its result */
 
 	snprintf(path, sizeof(path), "%s/%s", dir, off);
 	if (list_dir(&dentries, &dentrieslen, path) == 0) {
@@ -356,7 +355,7 @@ process_files(const char *dir, const char *off, FILE *m)
 			snprintf(path, sizeof(path), "%s%s%s",
 					off, *off == '\0' ? "" : "/", dentries[i]);
 			free(dentries[i]);
-			if (process_files(dir, path, m) == 0)
+			if (process_files(tv, dir, path, m) == 0)
 				continue;
 			/* regular file */
 			write_hashes(tv, dir, path, "AUX", m, NULL);
@@ -452,22 +451,22 @@ parse_layout_conf(const char *path)
 static char *str_manifest = "Manifest";
 static char *str_manifest_gz = "Manifest.gz";
 static char *str_manifest_files_gz = "Manifest.files.gz";
+enum type_manifest {
+	GLOBAL_MANIFEST,   /* Manifest.files.gz + Manifest */
+	SUBTREE_MANIFEST,  /* Manifest.gz for recursive list of files */
+	EBUILD_MANIFEST,   /* Manifest thick from thin */
+	CATEGORY_MANIFEST  /* Manifest.gz with Manifest entries */
+};
 static char *
-process_dir_gen(const char *dir)
+generate_dir(const char *dir, enum type_manifest mtype)
 {
-	char manifest[8192];
 	FILE *f;
 	char path[8192];
-	const char *p;
-	int newhashes;
-	enum {
-		GLOBAL_MANIFEST,   /* Manifest.files.gz + Manifest */
-		SUBTREE_MANIFEST,  /* Manifest.gz for recursive list of files */
-		EBUILD_MANIFEST,   /* Manifest thick from thin */
-		CATEGORY_MANIFEST  /* Manifest.gz with Manifest entries */
-	} type_manifest;
 	struct stat s;
 	struct timeval tv[2];
+	char **dentries;
+	size_t dentrieslen;
+	size_t i;
 
 	/* our timestamp strategy is as follows:
 	 * - when a Manifest exists, use its timestamp
@@ -482,155 +481,216 @@ process_dir_gen(const char *dir)
 	tv[1].tv_sec = 0;
 	tv[1].tv_usec = 0;
 
-	type_manifest = CATEGORY_MANIFEST;
-	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
-	if ((newhashes = parse_layout_conf(path)) != 0) {
-		type_manifest = GLOBAL_MANIFEST;
-		hashes = newhashes;
-	} else {
-		if ((p = strrchr(dir, '/')) != NULL) {
-			p++;
-		} else {
-			p = dir;
+	if (mtype == GLOBAL_MANIFEST) {
+		char *mfest;
+		size_t len;
+		gzFile mf;
+		time_t rtime;
+
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_files_gz);
+		if ((mf = gzopen(path, "wb9")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+					path, strerror(errno));
+			return NULL;
 		}
 
-		if (
-				strcmp(p, "eclass") == 0 ||
-				strcmp(p, "licenses") == 0 ||
-				strcmp(p, "metadata") == 0 ||
-				strcmp(p, "profiles") == 0 ||
-				strcmp(p, "scripts") == 0
-			)
-		{
-			type_manifest = SUBTREE_MANIFEST;
+		/* These "IGNORE" entries are taken from gx86, there is no
+		 * standardisation on this, on purpose, apparently. */
+		len = snprintf(path, sizeof(path),
+				"IGNORE distfiles\n"
+				"IGNORE local\n"
+				"IGNORE lost+found\n"
+				"IGNORE packages\n");
+		gzwrite(mf, path, len);
+
+		if (list_dir(&dentries, &dentrieslen, dir) != 0)
+			return NULL;
+
+		for (i = 0; i < dentrieslen; i++) {
+			/* ignore existing Manifests */
+			if (strcmp(dentries[i], str_manifest_files_gz) == 0 ||
+					strcmp(dentries[i], str_manifest) == 0)
+			{
+				free(dentries[i]);
+				continue;
+			}
+
+			snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
+
+			mfest = NULL;
+			if (!stat(path, &s)) {
+				if (s.st_mode & S_IFDIR) {
+					if (
+							strcmp(dentries[i], "eclass")   == 0 ||
+							strcmp(dentries[i], "licenses") == 0 ||
+							strcmp(dentries[i], "metadata") == 0 ||
+							strcmp(dentries[i], "profiles") == 0 ||
+							strcmp(dentries[i], "scripts")  == 0
+					   )
+					{
+						mfest = generate_dir(path, SUBTREE_MANIFEST);
+					} else {
+						mfest = generate_dir(path, CATEGORY_MANIFEST);
+					}
+
+					if (mfest == NULL) {
+						fprintf(stderr, "generating Manifest for %s failed!\n",
+								path);
+						gzclose(mf);
+						return NULL;
+					}
+
+					snprintf(path, sizeof(path), "%s/%s",
+							dentries[i], mfest);
+					write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
+				} else if (s.st_mode & S_IFREG) {
+					write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
+				} /* ignore other "things" (like symlinks) as they
+					 don't belong in a tree */
+			} else {
+				fprintf(stderr, "stat(%s) failed: %s\n",
+						path, strerror(errno));
+			}
+			free(dentries[i]);
 		}
-	}
+		free(dentries);
+		gzclose(mf);
 
-	/* If a Manifest file exists, this is an ebuild dir, unless we
-	 * already established this is the top level dir which also has a
-	 * Manifest file. */
-	snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest);
-	if (type_manifest == GLOBAL_MANIFEST ||
-			(f = fopen(manifest, "r")) == NULL)
-	{
-		/* all of these types (GLOBAL, SUBTREE, CATEGORY) have a gzipped
-		 * Manifest */
+		if (tv[0].tv_sec != 0) {
+			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_files_gz);
+			utimes(path, tv);
+		}
+
+		/* create global Manifest */
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
+		if ((f = fopen(path, "w")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+					path, strerror(errno));
+			return NULL;
+		}
+
+		write_hashes(tv, dir, str_manifest_files_gz, "MANIFEST", f, NULL);
+		time(&rtime);
+		len = strftime(path, sizeof(path),
+				"TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n", gmtime(&rtime));
+		fwrite(path, len, 1, f);
+		fflush(f);
+		fclose(f);
+
+		/* because we write a timestamp in Manifest, we don't mess with
+		 * its mtime, else it would obviously lie */
+		return str_manifest_files_gz;
+	} else if (mtype == SUBTREE_MANIFEST) {
+		const char *ldir;
 		gzFile mf;
-		char **dentries;
-		size_t dentrieslen;
-		size_t i;
 
-		if (list_dir(&dentries, &dentrieslen, dir) == 0) {
-			char *my_manifest = str_manifest_gz;
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
+		if ((mf = gzopen(path, "wb9")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+					path, strerror(errno));
+			return NULL;
+		}
 
-			if (type_manifest == GLOBAL_MANIFEST)
-				my_manifest = str_manifest_files_gz;
+		ldir = strrchr(dir, '/');
+		if (ldir == NULL)
+			ldir = dir;
+		if (strcmp(ldir, "metadata") == 0) {
+			size_t len;
+			len = snprintf(path, sizeof(path),
+					"IGNORE timestamp\n"
+					"IGNORE timestamp.chk\n"
+					"IGNORE timestamp.commit\n"
+					"IGNORE timestamp.x\n");
+			gzwrite(mf, path, len);
+		}
+
+		if (list_dir(&dentries, &dentrieslen, dir) != 0)
+			return NULL;
 
-			snprintf(manifest, sizeof(manifest), "%s/%s", dir, my_manifest);
-			if ((mf = gzopen(manifest, "wb9")) == NULL) {
-				fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-						manifest, strerror(errno));
-				return NULL;
+		for (i = 0; i < dentrieslen; i++) {
+			/* ignore existing Manifests */
+			if (strcmp(dentries[i], str_manifest_gz) == 0) {
+				free(dentries[i]);
+				continue;
 			}
 
-			for (i = 0; i < dentrieslen; i++) {
-				/* ignore existing Manifests */
-				if (strcmp(dentries[i], my_manifest) == 0 ||
-						strcmp(dentries[i], str_manifest) == 0)
-				{
-					free(dentries[i]);
-					continue;
-				}
+			if (write_hashes_dir(tv, dir, dentries[i], mf) != 0)
+				write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
+			free(dentries[i]);
+		}
 
-				snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
-				if (!stat(path, &s)) {
-					if (s.st_mode & S_IFDIR) {
-						if (type_manifest == SUBTREE_MANIFEST) {
-							write_hashes_dir(tv, dir, dentries[i], mf);
-							if (strcmp(dentries[i], "metadata") == 0) {
-								char buf[2048];
-								size_t len;
-								len = snprintf(buf, sizeof(buf),
-										"IGNORE timestamp\n"
-										"IGNORE timestamp.chk\n"
-										"IGNORE timestamp.commit\n"
-										"IGNORE timestamp.x\n");
-								gzwrite(mf, buf, len);
-							}
-							free(dentries[i]);
-						} else {
-							char *mfest = process_dir_gen(path);
-							if (mfest == NULL) {
-								gzclose(mf);
-								free(dentries[i]);
-								return NULL;
-							}
-							snprintf(path, sizeof(path), "%s/%s",
-									dentries[i], mfest);
-							free(dentries[i]);
-							write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
-						}
-					} else if (s.st_mode & S_IFREG) {
-						write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
-						free(dentries[i]);
-					}
-				}
+		free(dentries);
+		gzclose(mf);
+
+		if (tv[0].tv_sec != 0) {
+			/* set Manifest and dir mtime to most recent file found */
+			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
+			utimes(path, tv);
+			utimes(dir, tv);
+		}
+
+		return str_manifest_gz;
+	} else if (mtype == CATEGORY_MANIFEST) {
+		char *mfest;
+		gzFile mf;
+
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
+		if ((mf = gzopen(path, "wb9")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+					path, strerror(errno));
+			return NULL;
+		}
+
+		if (list_dir(&dentries, &dentrieslen, dir) != 0)
+			return NULL;
+
+		for (i = 0; i < dentrieslen; i++) {
+			/* ignore existing Manifests */
+			if (strcmp(dentries[i], str_manifest_gz) == 0) {
+				free(dentries[i]);
+				continue;
 			}
-			free(dentries);
 
-			if (type_manifest == GLOBAL_MANIFEST) {
-				char globmanifest[8192];
-				char buf[2048];
-				size_t len;
-				FILE *m;
-				time_t rtime;
-				struct timeval ntv[2]; /* dummy, not used */
-
-				len = snprintf(buf, sizeof(buf),
-						"IGNORE distfiles\n"
-						"IGNORE local\n"
-						"IGNORE lost+found\n"
-						"IGNORE packages\n");
-				gzwrite(mf, buf, len);
-				gzclose(mf);
-
-				/* create global Manifest */
-				snprintf(globmanifest, sizeof(globmanifest),
-						"%s/%s", dir, str_manifest);
-				if ((m = fopen(globmanifest, "w")) == NULL) {
-					fprintf(stderr, "failed to open file '%s' "
-							"for writing: %s\n",
-							globmanifest, strerror(errno));
-					return NULL;
-				}
+			snprintf(path, sizeof(path), "%s/%s", dir, dentries[i]);
+			if (!stat(path, &s)) {
+				if (s.st_mode & S_IFDIR) {
+					mfest = generate_dir(path, EBUILD_MANIFEST);
+
+					if (mfest == NULL) {
+						fprintf(stderr, "generating Manifest for %s failed!\n",
+								path);
+						gzclose(mf);
+						return NULL;
+					}
 
-				write_hashes(ntv, dir, my_manifest, "MANIFEST", m, NULL);
-				time(&rtime);
-				len = strftime(buf, sizeof(buf),
-						"TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n", gmtime(&rtime));
-				fwrite(buf, len, 1, m);
-				fflush(m);
-				fclose(m);
+					snprintf(path, sizeof(path), "%s/%s",
+							dentries[i], mfest);
+					write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
+				} else if (s.st_mode & S_IFREG) {
+					write_hashes(tv, dir, dentries[i], "DATA", NULL, mf);
+				} /* ignore other "things" (like symlinks) as they
+					 don't belong in a tree */
 			} else {
-				gzclose(mf);
+				fprintf(stderr, "stat(%s) failed: %s\n",
+						path, strerror(errno));
 			}
+			free(dentries[i]);
+		}
 
-			if (tv[0].tv_sec != 0) {
-				/* restore dir mtime, and set Manifest mtime to match it */
-				utimes(manifest, tv);
-				utimes(dir, tv);
-			}
+		free(dentries);
+		gzclose(mf);
+
+		if (tv[0].tv_sec != 0) {
+			/* set Manifest and dir mtime to most ebuild dir found */
+			snprintf(path, sizeof(path), "%s/%s", dir, str_manifest_gz);
+			utimes(path, tv);
+			utimes(dir, tv);
 		}
 
 		return str_manifest_gz;
-	} else {
-		/* this looks like an ebuild dir, so update the Manifest */
-		FILE *m;
+	} else if (mtype == EBUILD_MANIFEST) {
 		char newmanifest[8192];
-		char buf[8192];
-		char **dentries;
-		size_t dentrieslen;
-		size_t i;
+		FILE *m;
 
 		snprintf(newmanifest, sizeof(newmanifest), "%s/.Manifest.new", dir);
 		if ((m = fopen(newmanifest, "w")) == NULL) {
@@ -643,20 +703,30 @@ process_dir_gen(const char *dir)
 		 * prefixed with AUX, hence, if it exists, we need to do it
 		 * first */
 		snprintf(path, sizeof(path), "%s/files", dir);
-		process_files(path, "", m);
-
-		/* copy the DIST entries, we could do it unconditional, but this
-		 * way we can re-run without producing invalid Manifests */
-		while (fgets(buf, sizeof(buf), f) != NULL) {
-			if (strncmp(buf, "DIST ", 5) == 0)
-				if (fwrite(buf, strlen(buf), 1, m) != 1) {
-					fprintf(stderr, "failed to write to %s/.Manifest.new: %s\n",
-							dir, strerror(errno));
-					fclose(f);
-					return NULL;
-				}
+		process_files(tv, path, "", m);
+
+		/* the Manifest file may be missing in case there are no DIST
+		 * entries to be stored */
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
+		if (!stat(path, &s))
+			update_times(tv, &s);
+		f = fopen(path, "r");
+		if (f != NULL) {
+			/* copy the DIST entries, we could do it unconditional, but this
+			 * way we can re-run without producing invalid Manifests */
+			while (fgets(path, sizeof(path), f) != NULL) {
+				if (strncmp(path, "DIST ", 5) == 0)
+					if (fwrite(path, strlen(path), 1, m) != 1) {
+						fprintf(stderr, "failed to write to "
+								"%s/.Manifest.new: %s\n",
+								dir, strerror(errno));
+						fclose(f);
+						fclose(m);
+						return NULL;
+					}
+			}
+			fclose(f);
 		}
-		fclose(f);
 
 		if (list_dir(&dentries, &dentrieslen, dir) == 0) {
 			for (i = 0; i < dentrieslen; i++) {
@@ -678,25 +748,43 @@ process_dir_gen(const char *dir)
 		fflush(m);
 		fclose(m);
 
-		if (stat(manifest, &s)) {
-			tv[0].tv_sec = 0;
-			tv[0].tv_usec = 0;
-		} else {
-			tv[0].tv_sec = s.st_atim.tv_sec;
-			tv[0].tv_usec = s.st_atim.tv_nsec / 1000;
-			tv[1].tv_sec = s.st_mtim.tv_sec;
-			tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
-		}
+		snprintf(path, sizeof(path), "%s/%s", dir, str_manifest);
+		rename(newmanifest, path);
 
-		rename(newmanifest, manifest);
 		if (tv[0].tv_sec != 0) {
-			/* restore dir mtime, and set Manifest mtime to match it */
-			utimes(manifest, tv);
+			/* set Manifest and dir mtime to most recent file we found */
+			utimes(path, tv);
 			utimes(dir, tv);
 		}
 
 		return str_manifest;
+	} else {
+		return NULL;
+	}
+}
+
+static char *
+process_dir_gen(const char *dir)
+{
+	char path[8192];
+	int newhashes;
+
+	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
+	if ((newhashes = parse_layout_conf(path)) != 0) {
+		hashes = newhashes;
+	} else {
+		return "generation must be done on a full tree";
+	}
+
+	if (chdir(dir) != 0) {
+		fprintf(stderr, "cannot chdir() to %s: %s\n", dir, strerror(errno));
+		return "not a directory";
 	}
+
+	if (generate_dir(".\0", GLOBAL_MANIFEST) == NULL)
+		return "generation failed";
+
+	return NULL;
 }
 
 static char
@@ -1396,14 +1484,14 @@ main(int argc, char *argv[])
 	if (argc > 1) {
 		for (; arg < argc; arg++) {
 			rsn = runfunc(argv[arg]);
-			if (runfunc == &process_dir_vrfy && rsn != NULL) {
+			if (rsn != NULL) {
 				printf("%s\n", rsn);
 				ret |= 1;
 			}
 		}
 	} else {
 		rsn = runfunc(".");
-		if (runfunc == &process_dir_vrfy && rsn != NULL) {
+		if (rsn != NULL) {
 			printf("%s\n", rsn);
 			ret |= 1;
 		}

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     7fc3cf2b4baddc8b98c994b8ee024330d8f29956
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  1 16:36:28 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar  1 16:36:28 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=7fc3cf2b

scripts/rsync-generation/hashgen: report some stats

 scripts/rsync-generation/hashgen.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 56abd691a3..446299f92e 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -795,6 +795,10 @@ verify_gpg_sig(const char *path)
 	return ret;
 }
 
+static size_t checked_manifests = 0;
+static size_t checked_files = 0;
+static size_t failed_files = 0;
+
 static char
 verify_file(const char *dir, char *mfline, const char *mfest)
 {
@@ -849,11 +853,14 @@ verify_file(const char *dir, char *mfline, const char *mfest)
 		return 1;
 	}
 
+	checked_files++;
+
 	if (flen != fsize) {
 		printf("%s:%s:\n- file size mismatch\n"
 				"       got: %zd\n"
 				"  expected: %lld\n",
 				mfest, path, flen, fsize);
+		failed_files++;
 		return 1;
 	}
 
@@ -959,6 +966,7 @@ verify_file(const char *dir, char *mfline, const char *mfest)
 		ret = 1;
 	}
 
+	failed_files += ret;
 	return ret;
 }
 
@@ -1295,6 +1303,7 @@ verify_manifest(const char *dir, const char *manifest)
 	qsort(elems, elemslen, sizeof(elems[0]), compare_elems);
 	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
 	ret = verify_dir(dir, elems, elemslen, 0, buf + 2);
+	checked_manifests++;
 
 	while (elemslen-- > 0)
 		free(elems[elemslen]);
@@ -1309,6 +1318,11 @@ process_dir_vrfy(const char *dir)
 	char buf[8192];
 	int newhashes;
 	char *ret = NULL;
+	struct timeval startt;
+	struct timeval finisht;
+	double etime;
+
+	gettimeofday(&startt, NULL);
 
 	fprintf(stdout, "verifying %s...\n", dir);
 	snprintf(buf, sizeof(buf), "%s/metadata/layout.conf", dir);
@@ -1337,6 +1351,12 @@ process_dir_vrfy(const char *dir)
 	if (verify_manifest(".\0", str_manifest) != 0)
 		ret = "manifest verification failed";
 
+	gettimeofday(&finisht, NULL);
+
+	etime = ((double)((finisht.tv_sec - startt.tv_sec) * 1000000 +
+				finisht.tv_usec) - (double)startt.tv_usec) / 1000000.0;
+	printf("checked %zd Manifests, %zd files, %zd failures in %.02fs\n",
+			checked_manifests, checked_files, failed_files, etime);
 	return ret;
 }
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     4d518efa8955097c86305e18730ccc0709845c72
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  1 14:03:41 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar  1 14:03:41 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=4d518efa

scripts/rsync-generation/hashgen: make error messages more uniform

 scripts/rsync-generation/hashgen.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 226daa74df..56abd691a3 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1128,8 +1128,8 @@ verify_dir(
 					slash = strchr(entry, ' ');
 					if (slash != NULL)
 						*slash = '\0';
-					fprintf(stderr, "%s: missing %s file: %s\n",
-							mfest, etpe == 'M' ? "MANIFEST" : "DATA", entry);
+					printf("%s:%s:\n- %s file not found\n",
+							mfest, entry, etpe == 'M' ? "MANIFEST" : "DATA");
 					if (slash != NULL)
 						*slash = ' ';
 				}
@@ -1137,7 +1137,7 @@ verify_dir(
 			} else if (cmp > 0) {
 				/* dir has extra element */
 				ret |= 1;
-				fprintf(stderr, "%s: stray file not in Manifest: %s\n",
+				printf("%s:\n- found excess file: %s\n",
 						mfest, dentries[curdentry]);
 				curdentry++;
 			}

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     c7d6ecd8600f5a69f95427176e2f23a0b84e7e75
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  1 10:54:24 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Mar  1 10:54:24 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=c7d6ecd8

scripts/rsync-generation/hashgen: return when write_hashes would produce garbage

If we can't stat the file we're writing hashes for, we're not going to
produce anything useful and something up in the stack went wrong
already.

 scripts/rsync-generation/hashgen.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 833c8e7205..534d278d91 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -224,8 +224,10 @@ write_hashes(
 
 	snprintf(fname, sizeof(fname), "%s/%s", root, name);
 
-	if (stat(fname, &s) == 0)
-		update_times(tv, &s);
+	if (stat(fname, &s) != 0)
+		return;
+
+	update_times(tv, &s);
 
 	get_hashes(fname, sha256, sha512, whrlpl, blak2b, &flen);
 
@@ -1275,6 +1277,7 @@ process_dir_vrfy(const char *dir)
 {
 	char buf[8192];
 	int newhashes;
+	char *ret = NULL;
 
 	fprintf(stdout, "verifying %s...\n", dir);
 	snprintf(buf, sizeof(buf), "%s/metadata/layout.conf", dir);
@@ -1290,7 +1293,7 @@ process_dir_vrfy(const char *dir)
 	}
 
 	if (verify_gpg_sig(str_manifest) != 0)
-		return "gpg signature invalid";
+		ret = "gpg signature invalid";
 
 	/* verification goes like this:
 	 * - verify the signature of the top-level Manifest file (done
@@ -1301,9 +1304,9 @@ process_dir_vrfy(const char *dir)
 	 * - recurse into directories for which Manifest files are defined
 	 */
 	if (verify_manifest(".\0", str_manifest) != 0)
-		return "manifest verification failed";
+		ret = "manifest verification failed";
 
-	return NULL;
+	return ret;
 }
 
 int
@@ -1339,14 +1342,14 @@ main(int argc, char *argv[])
 	if (argc > 1) {
 		for (; arg < argc; arg++) {
 			rsn = runfunc(argv[arg]);
-			if (rsn != NULL) {
+			if (runfunc == &process_dir_vrfy && rsn != NULL) {
 				printf("%s\n", rsn);
 				ret |= 1;
 			}
 		}
 	} else {
 		rsn = runfunc(".");
-		if (rsn != NULL) {
+		if (runfunc == &process_dir_vrfy && rsn != NULL) {
 			printf("%s\n", rsn);
 			ret |= 1;
 		}

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     2fd3c876ab81f282671cb98fe924c92c329611a2
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 19:31:15 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 19:31:15 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=2fd3c876

scripts/rsync-generation/update-rsync-master: ensure mtime always moves forwards

relying on times stored in git is never going to work well, so pick a
time that we reasonably sync between multiple generators

 scripts/rsync-generation/update-rsync-master.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index ef30f387d5..86a9a4773b 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+SCRIPTSTARTTIME=$(date +%s)
+
 # get keys for ssh and signing
 eval $(env SHELL=/bin/bash keychain -q --noask --eval)
 
@@ -43,6 +45,15 @@ apply_git_mtimes() {
 	local from=$1
 	local to=$2
 
+	# As of 28-02-2018 we no longer take author or committer time,
+	# because both can be garbage (in the future, or terribly in the
+	# past).  Instead, we take the starttime of this script, rounded to
+	# the minute.  Because all generators should have this set off from
+	# cron at the same start-time, this should result in the trees
+	# staying in sync.  A scheduled synchronisation should wipe out any
+	# differences that may happen.
+	local thistime="$(((SCRIPTSTARTTIME / 60) * 60))"
+
 	local ts=0
 	local files=()
 	{
@@ -54,7 +65,7 @@ apply_git_mtimes() {
 			[0-9][0-9][0-9]*)
 				if [[ ${ts} -gt 0 ]] ; then
 					[[ ${#files[@]} == 0 ]] || \
-						touch -m -d @${ts} -- "${files[@]}"
+						touch -m -d @${thistime} -- "${files[@]}"
 				fi
 				ts=${line}
 				files=()

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     7b47cf64b5db7ce03f7a41e24b09182961444197
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 19:09:44 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 19:09:44 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=7b47cf64

script/rsync-generation/hashgen: allow compilation on Darwin

 scripts/rsync-generation/hashgen.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index b2ba8b0f5a..833c8e7205 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -99,6 +99,10 @@ hex_hash(char *out, const unsigned char *buf, const int length)
 static inline void
 update_times(struct timeval *tv, struct stat *s)
 {
+#ifdef __MACH__
+# define st_mtim st_mtimespec
+# define st_atim st_atimespec
+#endif
 	if (tv[1].tv_sec < s->st_mtim.tv_sec ||
 			(tv[1].tv_sec == s->st_mtim.tv_sec &&
 			 tv[1].tv_usec < s->st_mtim.tv_nsec / 1000))

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     d2ac9484a50843e9996df1f3f7dfd3eb0eb6ed85
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 16:29:10 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 16:29:10 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=d2ac9484

scripts/rsync-generation/hashgen: properly free members of lists

 scripts/rsync-generation/hashgen.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index a4df7ca850..3f7aaf65d6 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1093,7 +1093,11 @@ verify_dir(
 				curdentry++;
 			}
 		}
+
+		while (dentrieslen-- > 0)
+			free(dentries[dentrieslen]);
 		free(dentries);
+
 		return ret;
 	} else {
 		return 1;
@@ -1205,6 +1209,9 @@ verify_manifest(const char *dir, const char *manifest)
 	qsort(elems, elemslen, sizeof(elems[0]), compare_elems);
 	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
 	ret = verify_dir(dir, elems, elemslen, 0, buf + 2);
+
+	while (elemslen-- > 0)
+		free(elems[elemslen]);
 	free(elems);
 
 	return ret;
@@ -1213,7 +1220,6 @@ verify_manifest(const char *dir, const char *manifest)
 static char *
 process_dir_vrfy(const char *dir)
 {
-	char *ret = NULL;
 	char buf[8192];
 	int newhashes;
 
@@ -1222,8 +1228,7 @@ process_dir_vrfy(const char *dir)
 	if ((newhashes = parse_layout_conf(buf)) != 0) {
 		hashes = newhashes;
 	} else {
-		fprintf(stderr, "verification must be done on a full tree\n");
-		return "not on full tree";
+		return "verification must be done on a full tree";
 	}
 
 	if (chdir(dir) != 0) {
@@ -1232,7 +1237,7 @@ process_dir_vrfy(const char *dir)
 	}
 
 	if (verify_gpg_sig(str_manifest) != 0)
-		ret = "gpg signature invalid";
+		return "gpg signature invalid";
 
 	/* verification goes like this:
 	 * - verify the signature of the top-level Manifest file (done
@@ -1240,12 +1245,12 @@ process_dir_vrfy(const char *dir)
 	 * - read the contents of the Manifest file, and process the
 	 *   entries - verify them, check there are no files which shouldn't
 	 *   be there
-	 * - recurse into directories for which Manifest files are defined */
-
+	 * - recurse into directories for which Manifest files are defined
+	 */
 	if (verify_manifest(".\0", str_manifest) != 0)
-		ret = "manifest verification failed";
+		return "manifest verification failed";
 
-	return ret;
+	return NULL;
 }
 
 int

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     a11406710f526f2b28e68f5544e6cd9e47710058
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 18:43:31 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 18:43:31 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=a1140671

scripts/rsync-generation/hashgen: allow verification to be massively parallel

save up subdirs to handle so we can nicely loop over them in a for-loop
which OpenMP can cheaply parallelise for us (for free)

 scripts/rsync-generation/hashgen.c | 109 +++++++++++++++++++++++++++----------
 1 file changed, 79 insertions(+), 30 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 3f7aaf65d6..b2ba8b0f5a 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -919,6 +919,12 @@ compare_strings(const void *l, const void *r)
 
 static char verify_manifest(const char *dir, const char *manifest);
 
+struct subdir_workload {
+	size_t subdirlen;
+	size_t elemslen;
+	char **elems;
+};
+
 #define LISTSZ 64
 static char
 verify_dir(
@@ -940,6 +946,9 @@ verify_dir(
 	char etpe;
 	char ret = 0;
 	int cmp;
+	struct subdir_workload **subdir = NULL;
+	size_t subdirsize = 0;
+	size_t subdirlen = 0;
 
 	/* shortcut a single Manifest entry pointing to the same dir
 	 * (happens at top-level) */
@@ -1013,40 +1022,41 @@ verify_dir(
 			/* handle subdirs first */
 			if ((slash = strchr(entry, '/')) != NULL) {
 				size_t sublen = slash - entry;
-				char ndir[8192];
-
-				if (etpe == 'M') {
-					size_t skiplen = strlen(dir) + 1 + sublen;
-					/* sub-Manifest, we need to do a proper recurse */
-					slash = strrchr(entry, '/');  /* cannot be NULL */
-					snprintf(ndir, sizeof(ndir), "%s/%s", dir, entry);
-					ndir[skiplen] = '\0';
-					slash = strchr(ndir + skiplen + 1, ' ');
-					if (slash != NULL)  /* path should fit in ndir ... */
-						*slash = '\0';
-					if (verify_file(dir, entry, mfest) != 0 ||
-						verify_manifest(ndir, ndir + skiplen + 1) != 0)
-						ret |= 1;
-				} else {
-					int elemstart = curelem;
-					char **subelems = &elems[curelem];
-					/* collect all entries like this one (same subdir) into
-					 * a sub-list that we can verify */
+				int elemstart = curelem;
+				char **subelems = &elems[curelem];
+
+				/* collect all entries like this one (same subdir) into
+				 * a sub-list that we can verify */
+				curelem++;
+				while (curelem < elemslen &&
+						strncmp(entry, elems[curelem] + 2 + skippath,
+							sublen + 1) == 0)
 					curelem++;
-					while (curelem < elemslen &&
-							strncmp(entry, elems[curelem] + 2 + skippath,
-								sublen + 1) == 0)
-						curelem++;
-					snprintf(ndir, sizeof(ndir), "%s/%.*s", dir,
-							(int)sublen, elems[elemstart] + 2 + skippath);
-					ret |= verify_dir(ndir, subelems,
-							curelem - elemstart, skippath + sublen + 1, mfest);
-					curelem--; /* move back, see below */
+
+				if (subdirlen == subdirsize) {
+					subdirsize += LISTSZ;
+					subdir = realloc(subdir,
+							subdirsize * sizeof(subdir[0]));
+					if (subdir == NULL) {
+						fprintf(stderr, "out of memory\n");
+						return 1;
+					}
+				}
+				subdir[subdirlen] = malloc(sizeof(struct subdir_workload));
+				if (subdir[subdirlen] == NULL) {
+					fprintf(stderr, "out of memory\n");
+					return 1;
 				}
-				
+				subdir[subdirlen]->subdirlen = sublen;
+				subdir[subdirlen]->elemslen = curelem - elemstart;
+				subdir[subdirlen]->elems = subelems;
+				subdirlen++;
+
+				curelem--; /* move back, see below */
+
 				/* modify the last entry to be the subdir, such that we
 				 * can let the code below synchronise with dentries */
-				elems[curelem][2 + skippath + sublen] = '\0';
+				elems[curelem][2 + skippath + sublen] = ' ';
 				entry = elems[curelem] + 2 + skippath;
 				etpe = 'S';  /* flag this was a subdir */
 			}
@@ -1083,6 +1093,8 @@ verify_dir(
 						*slash = '\0';
 					fprintf(stderr, "%s: missing %s file: %s\n",
 							mfest, etpe == 'M' ? "MANIFEST" : "DATA", entry);
+					if (slash != NULL)
+						*slash = ' ';
 				}
 				curelem++;
 			} else if (cmp > 0) {
@@ -1098,6 +1110,43 @@ verify_dir(
 			free(dentries[dentrieslen]);
 		free(dentries);
 
+#pragma omp parallel for shared(ret) private(entry, etpe, slash)
+		for (cmp = 0; cmp < subdirlen; cmp++) {
+			char ndir[8192];
+
+			entry = subdir[cmp]->elems[0] + 2 + skippath;
+			etpe = subdir[cmp]->elems[0][0];
+
+			/* restore original entry format */
+			subdir[cmp]->elems[subdir[cmp]->elemslen - 1]
+				[2 + skippath + subdir[cmp]->subdirlen] = '/';
+
+			if (etpe == 'M') {
+				size_t skiplen = strlen(dir) + 1 + subdir[cmp]->subdirlen;
+				/* sub-Manifest, we need to do a proper recurse */
+				slash = strrchr(entry, '/');  /* cannot be NULL */
+				snprintf(ndir, sizeof(ndir), "%s/%s", dir, entry);
+				ndir[skiplen] = '\0';
+				slash = strchr(ndir + skiplen + 1, ' ');
+				if (slash != NULL)  /* path should fit in ndir ... */
+					*slash = '\0';
+				if (verify_file(dir, entry, mfest) != 0 ||
+						verify_manifest(ndir, ndir + skiplen + 1) != 0)
+					ret |= 1;
+			} else {
+				snprintf(ndir, sizeof(ndir), "%s/%.*s", dir,
+						(int)subdir[cmp]->subdirlen, entry);
+				ret |= verify_dir(ndir, subdir[cmp]->elems,
+						subdir[cmp]->elemslen,
+						skippath + subdir[cmp]->subdirlen + 1, mfest);
+			}
+
+			free(subdir[cmp]);
+		}
+
+		if (subdir)
+			free(subdir);
+
 		return ret;
 	} else {
 		return 1;

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     602608a0acadd589ec2ce85773aa5ec4ecfe228a
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 16:11:02 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 16:11:02 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=602608a0

scripts/rsync-generation/hashgen: try to print verification issues prettier

Still not very happy about this, need a clearer way to deal with issues
to make a readable conclusion.

 scripts/rsync-generation/hashgen.c | 95 +++++++++++++++++++++++++++-----------
 1 file changed, 67 insertions(+), 28 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 5eb7b8640b..a4df7ca850 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -725,7 +725,7 @@ verify_gpg_sig(const char *path)
 }
 
 static char
-verify_file(const char *dir, char *mfline)
+verify_file(const char *dir, char *mfline, const char *mfest)
 {
 	char *path;
 	char *size;
@@ -741,14 +741,15 @@ verify_file(const char *dir, char *mfline)
 	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
 	char ret = 0;
 
-	/* mfline is a Manifest file line with type stripped, something like:
-	 * path/to/file <SIZE> <HASHTYPE HASH ...>
+	/* mfline is a Manifest file line with type and leading path
+	 * stripped, something like:
+	 * file <SIZE> <HASHTYPE HASH ...>
 	 * we parse this, and verify the size and hashes */
 
 	path = mfline;
 	p = strchr(path, ' ');
 	if (p == NULL) {
-		fprintf(stderr, "%s: corrupt manifest line: %s\n", dir, path);
+		fprintf(stderr, "%s: corrupt manifest line: %s\n", mfest, path);
 		return 1;
 	}
 	*p++ = '\0';
@@ -757,14 +758,14 @@ verify_file(const char *dir, char *mfline)
 	p = strchr(size, ' ');
 	if (p == NULL) {
 		fprintf(stderr, "%s: corrupt manifest line, need size for %s\n",
-				dir, path);
+				mfest, path);
 		return 1;
 	}
 	*p++ = '\0';
 	fsize = strtoll(size, NULL, 10);
 	if (fsize == 0 && errno == EINVAL) {
 		fprintf(stderr, "%s: corrupt manifest line, size is not a number: %s\n",
-				dir, size);
+				dir + 2, size);
 		return 1;
 	}
 
@@ -773,13 +774,15 @@ verify_file(const char *dir, char *mfline)
 	get_hashes(buf, sha256, sha512, whrlpl, blak2b, &flen);
 
 	if (flen == 0) {
-		fprintf(stderr, "cannot locate %s\n", path);
+		fprintf(stderr, "cannot locate %s/%s\n", dir + 2, path);
 		return 1;
 	}
 
 	if (flen != fsize) {
-		fprintf(stderr, "%s: size mismatch, got: %zd, expected: %lld\n",
-				path, flen, fsize);
+		printf("%s:%s:\n- file size mismatch\n"
+				"       got: %zd\n"
+				"  expected: %lld\n",
+				mfest, path, flen, fsize);
 		return 1;
 	}
 
@@ -790,7 +793,7 @@ verify_file(const char *dir, char *mfline)
 		p = strchr(hashtype, ' ');
 		if (p == NULL) {
 			fprintf(stderr, "%s: corrupt manifest line, missing hash type\n",
-					path);
+					mfest);
 			return 1;
 		}
 		*p++ = '\0';
@@ -800,11 +803,14 @@ verify_file(const char *dir, char *mfline)
 		if (p != NULL)
 			*p++ = '\0';
 
+#define idif(X) if (X == 0) printf("%s:%s:\n", mfest, path);
 		if (strcmp(hashtype, "SHA256") == 0) {
 			if (!(hashes & HASH_SHA256)) {
+				idif(ret);
 				printf("- warning: hash SHA256 ignored as "
 						"it is not enabled for this repository\n");
 			} else if (strcmp(hash, sha256) != 0) {
+				idif(ret);
 				printf("- SHA256 hash mismatch\n"
 						"              computed: '%s'\n"
 						"  recorded in manifest: '%s'\n",
@@ -814,9 +820,11 @@ verify_file(const char *dir, char *mfline)
 			sha256[0] = '\0';
 		} else if (strcmp(hashtype, "SHA512") == 0) {
 			if (!(hashes & HASH_SHA512)) {
+				idif(ret);
 				printf("- warning: hash SHA512 ignored as "
 						"it is not enabled for this repository\n");
 			} else if (strcmp(hash, sha512) != 0) {
+				idif(ret);
 				printf("- SHA512 hash mismatch\n"
 						"              computed: '%s'\n"
 						"  recorded in manifest: '%s'\n",
@@ -826,9 +834,11 @@ verify_file(const char *dir, char *mfline)
 			sha512[0] = '\0';
 		} else if (strcmp(hashtype, "WHIRLPOOL") == 0) {
 			if (!(hashes & HASH_WHIRLPOOL)) {
+				idif(ret);
 				printf("- warning: hash WHIRLPOOL ignored as "
 						"it is not enabled for this repository\n");
 			} else if (strcmp(hash, whrlpl) != 0) {
+				idif(ret);
 				printf("- WHIRLPOOL hash mismatch\n"
 						"              computed: '%s'\n"
 						"  recorded in manifest: '%s'\n",
@@ -838,9 +848,11 @@ verify_file(const char *dir, char *mfline)
 			whrlpl[0] = '\0';
 		} else if (strcmp(hashtype, "BLAKE2B") == 0) {
 			if (!(hashes & HASH_BLAKE2B)) {
+				idif(ret);
 				printf("- warning: hash BLAKE2B ignored as "
 						"it is not enabled for this repository\n");
 			} else if (strcmp(hash, blak2b) != 0) {
+				idif(ret);
 				printf("- BLAKE2B hash mismatch\n"
 						"              computed: '%s'\n"
 						"  recorded in manifest: '%s'\n",
@@ -849,24 +861,29 @@ verify_file(const char *dir, char *mfline)
 			}
 			blak2b[0] = '\0';
 		} else {
+			idif(ret);
 			printf("- unsupported hash: %s\n", hashtype);
 			ret = 1;
 		}
 	}
 
 	if (sha256[0] != '\0') {
+		idif(ret);
 		printf("- missing hash: SHA256\n");
 		ret = 1;
 	}
 	if (sha512[0] != '\0') {
+		idif(ret);
 		printf("- missing hash: SHA512\n");
 		ret = 1;
 	}
 	if (whrlpl[0] != '\0') {
+		idif(ret);
 		printf("- missing hash: WHIRLPOOL\n");
 		ret = 1;
 	}
 	if (blak2b[0] != '\0') {
+		idif(ret);
 		printf("- missing hash: BLAKE2B\n");
 		ret = 1;
 	}
@@ -904,7 +921,12 @@ static char verify_manifest(const char *dir, const char *manifest);
 
 #define LISTSZ 64
 static char
-verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
+verify_dir(
+		const char *dir,
+		char **elems,
+		size_t elemslen,
+		size_t skippath,
+		const char *mfest)
 {
 	DIR *d;
 	struct dirent *e;
@@ -924,7 +946,7 @@ verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
 	if (elemslen == 1 && skippath == 0 &&
 			**elems == 'M' && strchr(*elems + 2, '/') == NULL)
 	{
-		if ((ret = verify_file(dir, *elems + 2)) == 0) {
+		if ((ret = verify_file(dir, *elems + 2, mfest)) == 0) {
 			slash = strchr(*elems + 2, ' ');
 			if (slash != NULL)
 				*slash = '\0';
@@ -997,13 +1019,12 @@ verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
 					size_t skiplen = strlen(dir) + 1 + sublen;
 					/* sub-Manifest, we need to do a proper recurse */
 					slash = strrchr(entry, '/');  /* cannot be NULL */
-					snprintf(ndir, sizeof(ndir),
-							"%s/%s", dir, entry);
+					snprintf(ndir, sizeof(ndir), "%s/%s", dir, entry);
 					ndir[skiplen] = '\0';
 					slash = strchr(ndir + skiplen + 1, ' ');
 					if (slash != NULL)  /* path should fit in ndir ... */
 						*slash = '\0';
-					if (verify_file(dir, entry) != 0 ||
+					if (verify_file(dir, entry, mfest) != 0 ||
 						verify_manifest(ndir, ndir + skiplen + 1) != 0)
 						ret |= 1;
 				} else {
@@ -1019,7 +1040,7 @@ verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
 					snprintf(ndir, sizeof(ndir), "%s/%.*s", dir,
 							(int)sublen, elems[elemstart] + 2 + skippath);
 					ret |= verify_dir(ndir, subelems,
-							curelem - elemstart, skippath + sublen + 1);
+							curelem - elemstart, skippath + sublen + 1, mfest);
 					curelem--; /* move back, see below */
 				}
 				
@@ -1045,7 +1066,7 @@ verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
 			if (cmp == 0) {
 				/* equal, so yay */
 				if (etpe == 'D') {
-					ret |= verify_file(dir, entry);
+					ret |= verify_file(dir, entry, mfest);
 				}
 				/* else this is I(GNORE) or S(ubdir), which means it is
 				 * ok in any way (M shouldn't happen) */
@@ -1061,14 +1082,14 @@ verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
 					if (slash != NULL)
 						*slash = '\0';
 					fprintf(stderr, "%s: missing %s file: %s\n",
-							dir, etpe == 'M' ? "MANIFEST" : "DATA", entry);
+							mfest, etpe == 'M' ? "MANIFEST" : "DATA", entry);
 				}
 				curelem++;
 			} else if (cmp > 0) {
 				/* dir has extra element */
 				ret |= 1;
 				fprintf(stderr, "%s: stray file not in Manifest: %s\n",
-						dir, dentries[curdentry]);
+						mfest, dentries[curdentry]);
 				curdentry++;
 			}
 		}
@@ -1085,6 +1106,7 @@ verify_manifest(const char *dir, const char *manifest)
 	char buf[8192];
 	FILE *f;
 	gzFile mf;
+	char ret = 0;
 
 	size_t elemssize = 0;
 	size_t elemslen = 0;
@@ -1181,10 +1203,11 @@ verify_manifest(const char *dir, const char *manifest)
 	 *   the larger Manifest.files.gz
 	 */
 	qsort(elems, elemslen, sizeof(elems[0]), compare_elems);
-	verify_dir(dir, elems, elemslen, 0);
+	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
+	ret = verify_dir(dir, elems, elemslen, 0, buf + 2);
 	free(elems);
 
-	return 0;
+	return ret;
 }
 
 static char *
@@ -1194,6 +1217,7 @@ process_dir_vrfy(const char *dir)
 	char buf[8192];
 	int newhashes;
 
+	fprintf(stdout, "verifying %s...\n", dir);
 	snprintf(buf, sizeof(buf), "%s/metadata/layout.conf", dir);
 	if ((newhashes = parse_layout_conf(buf)) != 0) {
 		hashes = newhashes;
@@ -1202,8 +1226,12 @@ process_dir_vrfy(const char *dir)
 		return "not on full tree";
 	}
 
-	snprintf(buf, sizeof(buf), "%s/%s", dir, str_manifest);
-	if (verify_gpg_sig(buf) != 0)
+	if (chdir(dir) != 0) {
+		fprintf(stderr, "cannot chdir() to %s: %s\n", dir, strerror(errno));
+		return "not a directory";
+	}
+
+	if (verify_gpg_sig(str_manifest) != 0)
 		ret = "gpg signature invalid";
 
 	/* verification goes like this:
@@ -1214,7 +1242,7 @@ process_dir_vrfy(const char *dir)
 	 *   be there
 	 * - recurse into directories for which Manifest files are defined */
 
-	if (verify_manifest(dir, str_manifest) != 0)
+	if (verify_manifest(".\0", str_manifest) != 0)
 		ret = "manifest verification failed";
 
 	return ret;
@@ -1226,6 +1254,8 @@ main(int argc, char *argv[])
 	char *prog;
 	char *(*runfunc)(const char *);
 	int arg = 1;
+	int ret = 0;
+	char *rsn;
 
 	if ((prog = strrchr(argv[0], '/')) == NULL)
 		prog = argv[0];
@@ -1249,11 +1279,20 @@ main(int argc, char *argv[])
 	gpgme_check_version(NULL);
 
 	if (argc > 1) {
-		for (; arg < argc; arg++)
-			runfunc(argv[arg]);
+		for (; arg < argc; arg++) {
+			rsn = runfunc(argv[arg]);
+			if (rsn != NULL) {
+				printf("%s\n", rsn);
+				ret |= 1;
+			}
+		}
 	} else {
-		runfunc(".");
+		rsn = runfunc(".");
+		if (rsn != NULL) {
+			printf("%s\n", rsn);
+			ret |= 1;
+		}
 	}
 
-	return 0;
+	return ret;
 }

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     ca1ef5e43e9810e10dd16bb3f6b94bf6ee6959c4
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 14:43:29 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 14:43:29 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=ca1ef5e4

scripts/rsync-generation/update-rsync-master: use -d with git-clean

also remove stray directories that for some reason where left behind, as
they cause verification problems

 scripts/rsync-generation/update-rsync-master.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 770e1b00c6..ef30f387d5 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -80,7 +80,7 @@ GLOBALSTART=${START}
 echo "($(date +"%F %R")) updating DTDs"
 pushd "$DTDDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git clean -fq
+git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -94,7 +94,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/dtd/timestamp.ch
 echo "($(date +"%F %R")) updating GLSAs"
 pushd "$GLSADIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git clean -fq
+git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -108,7 +108,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/glsa/timestamp.c
 echo "($(date +"%F %R")) updating news"
 pushd "$NEWSDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git clean -fq
+git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -136,7 +136,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating the gx86 tree"
 pushd "${GENTOOX86DIR}" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git clean -fq
+git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -162,7 +162,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating Prefix tree (Git image)"
 pushd "$PREFIXTREEDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
-git clean -fq
+git clean -dfq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     c68a2cd50ce42f42aaf785c16b6f7936c6b36bc3
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 11:40:52 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 11:40:52 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=c68a2cd5

hashgen: first shot at hashverify

This variant can verify the gx86 tree, or so it seems.

 scripts/rsync-generation/hashgen.c | 748 +++++++++++++++++++++++++++++++++++--
 1 file changed, 716 insertions(+), 32 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index fa0519fb04..5eb7b8640b 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1,5 +1,6 @@
-/* Copyright 2006-2017 Gentoo Foundation; Distributed under the GPL v2 */
+/* Copyright 2006-2018 Gentoo Foundation; Distributed under the GPL v2 */
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <strings.h>
 #include <ctype.h>
@@ -13,6 +14,7 @@
 #include <openssl/whrlpool.h>
 #include <blake2.h>
 #include <zlib.h>
+#include <gpgme.h>
 
 /* Generate thick Manifests based on thin Manifests */
 
@@ -20,8 +22,10 @@
  * - app-crypt/libb2 (for BLAKE2, for as long as openssl doesn't include it)
  * - dev-libs/openssl (for SHA, WHIRLPOOL)
  * - sys-libs/zlib (for compressing Manifest files)
- * compile like this
- *   ${CC} -o hashgen -fopenmp ${CFLAGS} -lssl -lcrypto -lb2 -lz hashgen.c
+ * - app-crypt/gpgme (for signing/verifying the top level manifest)
+ * compile like this:
+ *   ${CC} -o hashgen -fopenmp ${CFLAGS} \
+ *         -lssl -lcrypto -lb2 -lz `gpgme-config --libs` hashgen.c
  */
 
 enum hash_impls {
@@ -38,9 +42,57 @@ static int hashes = HASH_DEFAULT;
 static inline void
 hex_hash(char *out, const unsigned char *buf, const int length)
 {
-	int i;
-	for (i = 0; i < length; i++) {
-		snprintf(&out[i * 2], 3, "%02x", buf[i]);
+	switch (length) {
+		/* SHA256_DIGEST_LENGTH */
+		case 32:
+			snprintf(out, 64 + 1,
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x",
+					buf[ 0], buf[ 1], buf[ 2], buf[ 3], buf[ 4],
+					buf[ 5], buf[ 6], buf[ 7], buf[ 8], buf[ 9],
+					buf[10], buf[11], buf[12], buf[13], buf[14],
+					buf[15], buf[16], buf[17], buf[18], buf[19],
+					buf[20], buf[21], buf[22], buf[23], buf[24],
+					buf[25], buf[26], buf[27], buf[28], buf[29],
+					buf[30], buf[31]
+					);
+			break;
+		/* SHA512_DIGEST_LENGTH, WHIRLPOOL_DIGEST_LENGTH, BLAKE2B_OUTBYTES */
+		case 64:
+			snprintf(out, 128 + 1,
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					"%02x%02x%02x%02x",
+					buf[ 0], buf[ 1], buf[ 2], buf[ 3], buf[ 4],
+					buf[ 5], buf[ 6], buf[ 7], buf[ 8], buf[ 9],
+					buf[10], buf[11], buf[12], buf[13], buf[14],
+					buf[15], buf[16], buf[17], buf[18], buf[19],
+					buf[20], buf[21], buf[22], buf[23], buf[24],
+					buf[25], buf[26], buf[27], buf[28], buf[29],
+					buf[30], buf[31], buf[32], buf[33], buf[34],
+					buf[35], buf[36], buf[37], buf[38], buf[39],
+					buf[40], buf[41], buf[42], buf[43], buf[44],
+					buf[45], buf[46], buf[47], buf[48], buf[49],
+					buf[50], buf[51], buf[52], buf[53], buf[54],
+					buf[55], buf[56], buf[57], buf[58], buf[59],
+					buf[60], buf[61], buf[62], buf[63]
+					);
+			break;
+		/* fallback case, should never be necessary */
+		default:
+			{
+				int i;
+				for (i = 0; i < length; i++) {
+					snprintf(&out[i * 2], 3, "%02x", buf[i]);
+				}
+			}
+			break;
 	}
 }
 
@@ -59,43 +111,32 @@ update_times(struct timeval *tv, struct stat *s)
 }
 
 static void
-write_hashes(
-		struct timeval *tv,
-		const char *root,
-		const char *name,
-		const char *type,
-		FILE *m,
-		gzFile gm)
+get_hashes(
+		const char *fname,
+		char *sha256,
+		char *sha512,
+		char *whrlpl,
+		char *blak2b,
+		size_t *flen)
 {
 	FILE *f;
-	char fname[8192];
-	size_t flen = 0;
-	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
-	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
-	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
-	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
 	char data[8192];
 	size_t len;
 	SHA256_CTX s256;
 	SHA512_CTX s512;
 	WHIRLPOOL_CTX whrl;
 	blake2b_state bl2b;
-	struct stat s;
 
-	snprintf(fname, sizeof(fname), "%s/%s", root, name);
 	if ((f = fopen(fname, "r")) == NULL)
 		return;
 
-	if (stat(fname, &s) == 0)
-		update_times(tv, &s);
-
 	SHA256_Init(&s256);
 	SHA512_Init(&s512);
 	WHIRLPOOL_Init(&whrl);
 	blake2b_init(&bl2b, BLAKE2B_OUTBYTES);
 
 	while ((len = fread(data, 1, sizeof(data), f)) > 0) {
-		flen += len;
+		*flen += len;
 #pragma omp parallel sections
 		{
 #pragma omp section
@@ -120,6 +161,7 @@ write_hashes(
 			}
 		}
 	}
+	fclose(f);
 
 #pragma omp parallel sections
 	{
@@ -155,7 +197,33 @@ write_hashes(
 			}
 		}
 	}
-	fclose(f);
+}
+
+static void
+write_hashes(
+		struct timeval *tv,
+		const char *root,
+		const char *name,
+		const char *type,
+		FILE *m,
+		gzFile gm)
+{
+	size_t flen = 0;
+	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
+	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
+	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
+	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
+	char data[8192];
+	char fname[8192];
+	size_t len;
+	struct stat s;
+
+	snprintf(fname, sizeof(fname), "%s/%s", root, name);
+
+	if (stat(fname, &s) == 0)
+		update_times(tv, &s);
+
+	get_hashes(fname, sha256, sha512, whrlpl, blak2b, &flen);
 
 	len = snprintf(data, sizeof(data), "%s %s %zd", type, name, flen);
 	if (hashes & HASH_BLAKE2B)
@@ -321,7 +389,7 @@ static char *str_manifest = "Manifest";
 static char *str_manifest_gz = "Manifest.gz";
 static char *str_manifest_files_gz = "Manifest.files.gz";
 static char *
-process_dir(const char *dir)
+process_dir_gen(const char *dir)
 {
 	char manifest[8192];
 	FILE *f;
@@ -426,7 +494,7 @@ process_dir(const char *dir)
 								gzwrite(mf, buf, len);
 							}
 						} else {
-							char *mfest = process_dir(path);
+							char *mfest = process_dir_gen(path);
 							if (mfest == NULL) {
 								gzclose(mf);
 								return NULL;
@@ -560,15 +628,631 @@ process_dir(const char *dir)
 	}
 }
 
+static char
+verify_gpg_sig(const char *path)
+{
+	gpgme_ctx_t g_ctx;
+	gpgme_data_t manifest;
+	gpgme_data_t out;
+	gpgme_verify_result_t vres;
+	gpgme_signature_t sig;
+	gpgme_key_t key;
+	char buf[32];
+	FILE *f;
+	struct tm *ctime;
+	char ret = 1;  /* fail */
+
+	if ((f = fopen(path, "r")) == NULL) {
+		fprintf(stderr, "failed to open %s: %s\n", path, strerror(errno));
+		return ret;
+	}
+
+	if (gpgme_new(&g_ctx) != GPG_ERR_NO_ERROR) {
+		fprintf(stderr, "failed to create gpgme context\n");
+		return ret;
+	}
+
+	if (gpgme_data_new(&out) != GPG_ERR_NO_ERROR) {
+		fprintf(stderr, "failed to create new gpgme data\n");
+		return ret;
+	}
+
+	if (gpgme_data_new_from_stream(&manifest, f) != GPG_ERR_NO_ERROR) {
+		fprintf(stderr, "failed to create new gpgme data from stream\n");
+		return ret;
+	}
+
+	if (gpgme_op_verify(g_ctx, manifest, NULL, out) != GPG_ERR_NO_ERROR) {
+		fprintf(stderr, "failed to verify signature\n");
+		return ret;
+	}
+
+	vres = gpgme_op_verify_result(g_ctx);
+	fclose(f);
+
+	for (sig = vres->signatures; sig != NULL; sig = sig->next) {
+		ctime = gmtime((time_t *)&sig->timestamp);
+		strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", ctime);
+		printf("%s key fingerprint "
+				"%.4s %.4s %.4s %.4s %.4s  %.4s %.4s %.4s %.4s %.4s\n"
+				"%s signature made %s by\n",
+				gpgme_pubkey_algo_name(sig->pubkey_algo),
+				sig->fpr +  0, sig->fpr +  4, sig->fpr +  8, sig->fpr + 12,
+				sig->fpr + 16, sig->fpr + 20, sig->fpr + 24, sig->fpr + 28,
+				sig->fpr + 32, sig->fpr + 36,
+				sig->status == GPG_ERR_NO_ERROR ? "good" : "BAD",
+				buf);
+
+		if (gpgme_get_key(g_ctx, sig->fpr, &key, 0) == GPG_ERR_NO_ERROR) {
+			ret = 0;  /* valid */
+			if (key->uids != NULL)
+				printf("%s\n", key->uids->uid);
+			gpgme_key_release(key);
+		}
+
+		switch (sig->status) {
+			case GPG_ERR_NO_ERROR:
+				/* nothing, handled above */
+				break;
+			case GPG_ERR_SIG_EXPIRED:
+				printf("the signature is valid but expired\n");
+				break;
+			case GPG_ERR_KEY_EXPIRED:
+				printf("the signature is valid but the key used to verify "
+						"the signature has expired\n");
+				break;
+			case GPG_ERR_CERT_REVOKED:
+				printf("the signature is valid but the key used to verify "
+						"the signature has been revoked\n");
+				break;
+			case GPG_ERR_BAD_SIGNATURE:
+				printf("the signature is invalid\n");
+				break;
+			case GPG_ERR_NO_PUBKEY:
+				printf("the signature could not be verified due to a "
+						"missing key\n");
+				break;
+			default:
+				printf("there was some other error which prevented the "
+						"signature verification\n");
+				break;
+		}
+	}
+
+	gpgme_release(g_ctx);
+
+	return ret;
+}
+
+static char
+verify_file(const char *dir, char *mfline)
+{
+	char *path;
+	char *size;
+	long long int fsize;
+	char *hashtype;
+	char *hash;
+	char *p;
+	char buf[8192];
+	size_t flen = 0;
+	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
+	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
+	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
+	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
+	char ret = 0;
+
+	/* mfline is a Manifest file line with type stripped, something like:
+	 * path/to/file <SIZE> <HASHTYPE HASH ...>
+	 * we parse this, and verify the size and hashes */
+
+	path = mfline;
+	p = strchr(path, ' ');
+	if (p == NULL) {
+		fprintf(stderr, "%s: corrupt manifest line: %s\n", dir, path);
+		return 1;
+	}
+	*p++ = '\0';
+
+	size = p;
+	p = strchr(size, ' ');
+	if (p == NULL) {
+		fprintf(stderr, "%s: corrupt manifest line, need size for %s\n",
+				dir, path);
+		return 1;
+	}
+	*p++ = '\0';
+	fsize = strtoll(size, NULL, 10);
+	if (fsize == 0 && errno == EINVAL) {
+		fprintf(stderr, "%s: corrupt manifest line, size is not a number: %s\n",
+				dir, size);
+		return 1;
+	}
+
+	sha256[0] = sha512[0] = whrlpl[0] = blak2b[0] = '\0';
+	snprintf(buf, sizeof(buf), "%s/%s", dir, path);
+	get_hashes(buf, sha256, sha512, whrlpl, blak2b, &flen);
+
+	if (flen == 0) {
+		fprintf(stderr, "cannot locate %s\n", path);
+		return 1;
+	}
+
+	if (flen != fsize) {
+		fprintf(stderr, "%s: size mismatch, got: %zd, expected: %lld\n",
+				path, flen, fsize);
+		return 1;
+	}
+
+	/* now we are in free territory, we read TYPE HASH pairs until we
+	 * drained the string, and match them against what we computed */
+	while (p != NULL && *p != '\0') {
+		hashtype = p;
+		p = strchr(hashtype, ' ');
+		if (p == NULL) {
+			fprintf(stderr, "%s: corrupt manifest line, missing hash type\n",
+					path);
+			return 1;
+		}
+		*p++ = '\0';
+
+		hash = p;
+		p = strchr(hash, ' ');
+		if (p != NULL)
+			*p++ = '\0';
+
+		if (strcmp(hashtype, "SHA256") == 0) {
+			if (!(hashes & HASH_SHA256)) {
+				printf("- warning: hash SHA256 ignored as "
+						"it is not enabled for this repository\n");
+			} else if (strcmp(hash, sha256) != 0) {
+				printf("- SHA256 hash mismatch\n"
+						"              computed: '%s'\n"
+						"  recorded in manifest: '%s'\n",
+						sha256, hash);
+				ret = 1;
+			}
+			sha256[0] = '\0';
+		} else if (strcmp(hashtype, "SHA512") == 0) {
+			if (!(hashes & HASH_SHA512)) {
+				printf("- warning: hash SHA512 ignored as "
+						"it is not enabled for this repository\n");
+			} else if (strcmp(hash, sha512) != 0) {
+				printf("- SHA512 hash mismatch\n"
+						"              computed: '%s'\n"
+						"  recorded in manifest: '%s'\n",
+						sha512, hash);
+				ret = 1;
+			}
+			sha512[0] = '\0';
+		} else if (strcmp(hashtype, "WHIRLPOOL") == 0) {
+			if (!(hashes & HASH_WHIRLPOOL)) {
+				printf("- warning: hash WHIRLPOOL ignored as "
+						"it is not enabled for this repository\n");
+			} else if (strcmp(hash, whrlpl) != 0) {
+				printf("- WHIRLPOOL hash mismatch\n"
+						"              computed: '%s'\n"
+						"  recorded in manifest: '%s'\n",
+						whrlpl, hash);
+				ret = 1;
+			}
+			whrlpl[0] = '\0';
+		} else if (strcmp(hashtype, "BLAKE2B") == 0) {
+			if (!(hashes & HASH_BLAKE2B)) {
+				printf("- warning: hash BLAKE2B ignored as "
+						"it is not enabled for this repository\n");
+			} else if (strcmp(hash, blak2b) != 0) {
+				printf("- BLAKE2B hash mismatch\n"
+						"              computed: '%s'\n"
+						"  recorded in manifest: '%s'\n",
+						blak2b, hash);
+				ret = 1;
+			}
+			blak2b[0] = '\0';
+		} else {
+			printf("- unsupported hash: %s\n", hashtype);
+			ret = 1;
+		}
+	}
+
+	if (sha256[0] != '\0') {
+		printf("- missing hash: SHA256\n");
+		ret = 1;
+	}
+	if (sha512[0] != '\0') {
+		printf("- missing hash: SHA512\n");
+		ret = 1;
+	}
+	if (whrlpl[0] != '\0') {
+		printf("- missing hash: WHIRLPOOL\n");
+		ret = 1;
+	}
+	if (blak2b[0] != '\0') {
+		printf("- missing hash: BLAKE2B\n");
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static int
+compare_elems(const void *l, const void *r)
+{
+	const char *strl = *((const char **)l) + 2;
+	const char *strr = *((const char **)r) + 2;
+	unsigned char cl;
+	unsigned char cr;
+	/* compare treating / as end of string */
+	while ((cl = *strl++) == (cr = *strr++))
+		if (cl == '\0')
+			return 0;
+	if (cl == '/')
+		cl = '\0';
+	if (cr == '/')
+		cr = '\0';
+	return cl - cr;
+}
+
+static int
+compare_strings(const void *l, const void *r)
+{
+	const char **strl = (const char **)l;
+	const char **strr = (const char **)r;
+	return strcmp(*strl, *strr);
+}
+
+static char verify_manifest(const char *dir, const char *manifest);
+
+#define LISTSZ 64
+static char
+verify_dir(const char *dir, char **elems, size_t elemslen, size_t skippath)
+{
+	DIR *d;
+	struct dirent *e;
+	char **dentries = NULL;
+	size_t dentrieslen = 0;
+	size_t dentriessize = 0;
+	size_t curelem = 0;
+	size_t curdentry = 0;
+	char *entry;
+	char *slash;
+	char etpe;
+	char ret = 0;
+	int cmp;
+
+	/* shortcut a single Manifest entry pointing to the same dir
+	 * (happens at top-level) */
+	if (elemslen == 1 && skippath == 0 &&
+			**elems == 'M' && strchr(*elems + 2, '/') == NULL)
+	{
+		if ((ret = verify_file(dir, *elems + 2)) == 0) {
+			slash = strchr(*elems + 2, ' ');
+			if (slash != NULL)
+				*slash = '\0';
+			/* else, verify_manifest will fail, so ret will be handled */
+			ret = verify_manifest(dir, *elems + 2);
+		}
+		return ret;
+	}
+
+	/*
+	 * We have a list of entries from the manifest just read, now we
+	 * need to match these onto the directory layout.  From what we got
+	 * - we can ignore TIMESTAMP and DIST entries
+	 * - IGNOREs need to be handled separate (shortcut)
+	 * - MANIFESTs need to be handled on their own, for memory
+	 *   consumption reasons, we defer them to until we've verified
+	 *   what's left, we treat the path the Manifest refers to as IGNORE
+	 * - DATAs, EBUILDs and MISCs needs verifying
+	 * - AUXs need verifying, but in files/ subdir
+	 * If we sort both lists, we should be able to do a merge-join, to
+	 * easily flag missing entries in either list without hashing or
+	 * anything.
+	 */
+	if ((d = opendir(dir)) != NULL) {
+		while ((e = readdir(d)) != NULL) {
+			/* skip all dotfiles and Manifest files */
+			if (e->d_name[0] == '.' ||
+					strcmp(e->d_name, str_manifest) == 0 ||
+					strcmp(e->d_name, str_manifest_gz) == 0 ||
+					strcmp(e->d_name, str_manifest_files_gz) == 0)
+			{
+				continue;
+			}
+
+			if (dentrieslen == dentriessize) {
+				dentriessize += LISTSZ;
+				dentries = realloc(dentries,
+						dentriessize * sizeof(dentries[0]));
+				if (dentries == NULL) {
+					fprintf(stderr, "out of memory\n");
+					return 1;
+				}
+			}
+			dentries[dentrieslen] = strdup(e->d_name);
+			if (dentries[dentrieslen] == NULL) {
+				fprintf(stderr, "out of memory\n");
+				return 1;
+			}
+			dentrieslen++;
+		}
+		closedir(d);
+
+		qsort(dentries, dentrieslen, sizeof(dentries[0]), compare_strings);
+
+		while (curdentry < dentrieslen) {
+			if (curelem < elemslen) {
+				entry = elems[curelem] + 2 + skippath;
+				etpe = *elems[curelem];
+			} else {
+				entry = "";
+				etpe = 'I';
+			}
+
+			/* handle subdirs first */
+			if ((slash = strchr(entry, '/')) != NULL) {
+				size_t sublen = slash - entry;
+				char ndir[8192];
+
+				if (etpe == 'M') {
+					size_t skiplen = strlen(dir) + 1 + sublen;
+					/* sub-Manifest, we need to do a proper recurse */
+					slash = strrchr(entry, '/');  /* cannot be NULL */
+					snprintf(ndir, sizeof(ndir),
+							"%s/%s", dir, entry);
+					ndir[skiplen] = '\0';
+					slash = strchr(ndir + skiplen + 1, ' ');
+					if (slash != NULL)  /* path should fit in ndir ... */
+						*slash = '\0';
+					if (verify_file(dir, entry) != 0 ||
+						verify_manifest(ndir, ndir + skiplen + 1) != 0)
+						ret |= 1;
+				} else {
+					int elemstart = curelem;
+					char **subelems = &elems[curelem];
+					/* collect all entries like this one (same subdir) into
+					 * a sub-list that we can verify */
+					curelem++;
+					while (curelem < elemslen &&
+							strncmp(entry, elems[curelem] + 2 + skippath,
+								sublen + 1) == 0)
+						curelem++;
+					snprintf(ndir, sizeof(ndir), "%s/%.*s", dir,
+							(int)sublen, elems[elemstart] + 2 + skippath);
+					ret |= verify_dir(ndir, subelems,
+							curelem - elemstart, skippath + sublen + 1);
+					curelem--; /* move back, see below */
+				}
+				
+				/* modify the last entry to be the subdir, such that we
+				 * can let the code below synchronise with dentries */
+				elems[curelem][2 + skippath + sublen] = '\0';
+				entry = elems[curelem] + 2 + skippath;
+				etpe = 'S';  /* flag this was a subdir */
+			}
+
+			/* does this entry exist in list? */
+			if (*entry == '\0') {
+				/* end of list reached, force dir to catch up */
+				cmp = 1;
+			} else {
+				slash = strchr(entry, ' ');
+				if (slash != NULL)
+					*slash = '\0';
+				cmp = strcmp(entry, dentries[curdentry]);
+				if (slash != NULL)
+					*slash = ' ';
+			}
+			if (cmp == 0) {
+				/* equal, so yay */
+				if (etpe == 'D') {
+					ret |= verify_file(dir, entry);
+				}
+				/* else this is I(GNORE) or S(ubdir), which means it is
+				 * ok in any way (M shouldn't happen) */
+				curelem++;
+				curdentry++;
+			} else if (cmp < 0) {
+				/* entry is missing from dir */
+				if (etpe == 'I') {
+					/* right, we can ignore this */
+				} else {
+					ret |= 1;
+					slash = strchr(entry, ' ');
+					if (slash != NULL)
+						*slash = '\0';
+					fprintf(stderr, "%s: missing %s file: %s\n",
+							dir, etpe == 'M' ? "MANIFEST" : "DATA", entry);
+				}
+				curelem++;
+			} else if (cmp > 0) {
+				/* dir has extra element */
+				ret |= 1;
+				fprintf(stderr, "%s: stray file not in Manifest: %s\n",
+						dir, dentries[curdentry]);
+				curdentry++;
+			}
+		}
+		free(dentries);
+		return ret;
+	} else {
+		return 1;
+	}
+}
+
+static char
+verify_manifest(const char *dir, const char *manifest)
+{
+	char buf[8192];
+	FILE *f;
+	gzFile mf;
+
+	size_t elemssize = 0;
+	size_t elemslen = 0;
+	char **elems = NULL;
+#define append_list(STR) \
+	if (strncmp(STR, "TIMESTAMP ", 10) != 0 || strncmp(STR, "DIST ", 5) != 0) {\
+		char *endp = STR + strlen(STR) - 1;\
+		while (isspace(*endp))\
+			*endp-- = '\0';\
+		if (elemslen == elemssize) {\
+			elemssize += LISTSZ;\
+			elems = realloc(elems, elemssize * sizeof(elems[0]));\
+			if (elems == NULL) {\
+				fprintf(stderr, "out of memory\n");\
+				return 1;\
+			}\
+		}\
+		if (strncmp(STR, "IGNORE ", 7) == 0) {\
+			STR[5] = 'I';\
+			elems[elemslen] = strdup(STR + 5);\
+			if (elems[elemslen] == NULL) {\
+				fprintf(stderr, "out of memory\n");\
+				return 1;\
+			}\
+			elemslen++;\
+		} else if (strncmp(STR, "MANIFEST ", 9) == 0) {\
+			STR[7] = 'M';\
+			elems[elemslen] = strdup(STR + 7);\
+			if (elems[elemslen] == NULL) {\
+				fprintf(stderr, "out of memory\n");\
+				return 1;\
+			}\
+			elemslen++;\
+		} else if (strncmp(STR, "DATA ", 5) == 0 ||\
+				strncmp(STR, "MISC ", 5) == 0 ||\
+				strncmp(STR, "EBUILD ", 7) == 0)\
+		{\
+			if (*STR == 'E') {\
+				STR[5] = 'D';\
+				elems[elemslen] = strdup(STR + 5);\
+			} else {\
+				STR[3] = 'D';\
+				elems[elemslen] = strdup(STR + 3);\
+			}\
+			if (elems[elemslen] == NULL) {\
+				fprintf(stderr, "out of memory\n");\
+				return 1;\
+			}\
+			elemslen++;\
+		} else if (strncmp(STR, "AUX ", 4) == 0) {\
+			/* translate directly into what it is: DATA in files/ */\
+			size_t slen = strlen(STR + 2) + sizeof("files/");\
+			elems[elemslen] = malloc(slen);\
+			if (elems[elemslen] == NULL) {\
+				fprintf(stderr, "out of memory\n");\
+				return 1;\
+			}\
+			snprintf(elems[elemslen], slen, "D files/%s", STR + 4);\
+			elemslen++;\
+		}\
+	}
+
+	snprintf(buf, sizeof(buf), "%s/%s", dir, manifest);
+	if (strcmp(manifest, str_manifest) == 0) {
+		if ((f = fopen(buf, "r")) == NULL) {
+			fprintf(stderr, "failed to open %s: %s\n",
+					buf, strerror(errno));
+			return 1;
+		}
+		while (fgets(buf, sizeof(buf), f) != NULL) {
+			append_list(buf);
+		}
+		fclose(f);
+	} else if (strcmp(manifest, str_manifest_files_gz) == 0 ||
+			strcmp(manifest, str_manifest_gz) == 0)
+	{
+		if ((mf = gzopen(buf, "rb9")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for reading: %s\n",
+					buf, strerror(errno));
+			return 1;
+		}
+		while (gzgets(mf, buf, sizeof(buf)) != NULL) {
+			append_list(buf);
+		}
+		gzclose(mf);
+	}
+
+	/* The idea:
+	 * - Manifest without MANIFEST entries, we need to scan the entire
+	 *   subtree
+	 * - Manifest with MANIFEST entries, assume they are just one level
+	 *   deeper, thus ignore that subdir, further like above
+	 * - Manifest at top-level, needs to be igored as it only points to
+	 *   the larger Manifest.files.gz
+	 */
+	qsort(elems, elemslen, sizeof(elems[0]), compare_elems);
+	verify_dir(dir, elems, elemslen, 0);
+	free(elems);
+
+	return 0;
+}
+
+static char *
+process_dir_vrfy(const char *dir)
+{
+	char *ret = NULL;
+	char buf[8192];
+	int newhashes;
+
+	snprintf(buf, sizeof(buf), "%s/metadata/layout.conf", dir);
+	if ((newhashes = parse_layout_conf(buf)) != 0) {
+		hashes = newhashes;
+	} else {
+		fprintf(stderr, "verification must be done on a full tree\n");
+		return "not on full tree";
+	}
+
+	snprintf(buf, sizeof(buf), "%s/%s", dir, str_manifest);
+	if (verify_gpg_sig(buf) != 0)
+		ret = "gpg signature invalid";
+
+	/* verification goes like this:
+	 * - verify the signature of the top-level Manifest file (done
+	 *   above)
+	 * - read the contents of the Manifest file, and process the
+	 *   entries - verify them, check there are no files which shouldn't
+	 *   be there
+	 * - recurse into directories for which Manifest files are defined */
+
+	if (verify_manifest(dir, str_manifest) != 0)
+		ret = "manifest verification failed";
+
+	return ret;
+}
+
 int
 main(int argc, char *argv[])
 {
+	char *prog;
+	char *(*runfunc)(const char *);
+	int arg = 1;
+
+	if ((prog = strrchr(argv[0], '/')) == NULL)
+		prog = argv[0];
+
+	if (argc > 1) {
+		if (strcmp(argv[1], "hashverify") == 0 ||
+				strcmp(argv[1], "hashgen") == 0)
+		{
+			prog = argv[1];
+			arg = 2;
+		}
+	}
+
+	if (strcmp(prog, "hashverify") == 0) {
+		runfunc = &process_dir_vrfy;
+	} else {
+		/* default mode: hashgen */
+		runfunc = &process_dir_gen;
+	}
+
+	gpgme_check_version(NULL);
+
 	if (argc > 1) {
-		int i;
-		for (i = 1; i < argc; i++)
-			process_dir(argv[i]);
+		for (; arg < argc; arg++)
+			runfunc(argv[arg]);
 	} else {
-		process_dir(".");
+		runfunc(".");
 	}
 
 	return 0;

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     fa562cfdd0ac11982e872509755a2a4cc668fe30
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 22 19:45:41 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Feb 22 19:45:41 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=fa562cfd

scripts/rsync-generation/update-rsync-master: don't bother egencache with missing digests

 scripts/rsync-generation/update-rsync-master.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index e225353098..770e1b00c6 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -193,8 +193,6 @@ touch -r "${GENTOOX86DIR}"/profiles/repo_name "${RSYNCDIR}"/profiles/repo_name
 # reset Prefix profiles to dev status
 sed -i -e '/prefix/s/exp/dev/' "${RSYNCDIR}"/profiles/profiles.desc
 touch -r "${GENTOOX86DIR}"/profiles/profiles.desc "${RSYNCDIR}"/profiles/profiles.desc
-# we will generate thick manifests, so ensure Portage knows that
-sed -i -e '/^thin-manifests/s/true/false/' "${RSYNCDIR}"/metadata/layout.conf
 echo "($(date +"%F %R")) set up repo $(< "${RSYNCDIR}"/profiles/repo_name)"
 
 
@@ -242,6 +240,9 @@ START=$(date +%s)
 
 echo "($(date +"%F %R")) signing Manifest"
 
+# we will generate thick manifests, so ensure Portage knows that
+sed -i -e '/^thin-manifests/s/true/false/' "${RSYNCDIR}"/metadata/layout.conf
+
 # generate Thick Manifests
 ${BASE_PATH}/hashgen "${RSYNCDIR}"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     8ac43f859bfccf89b9c211a611f965b035de8ecd
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 22 07:28:20 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Feb 22 07:28:20 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=8ac43f85

scripts/rsync-generation/update-rsync-master: set thin-manifests to false

When using Git, we use thin-manifests, but on the rsync tree, we
transformed the manifests into fat ones (hashgen), so we need to ensure
Portage knows that and performs its checks.

 scripts/rsync-generation/update-rsync-master.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 0a4cf32d15..e225353098 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -193,6 +193,8 @@ touch -r "${GENTOOX86DIR}"/profiles/repo_name "${RSYNCDIR}"/profiles/repo_name
 # reset Prefix profiles to dev status
 sed -i -e '/prefix/s/exp/dev/' "${RSYNCDIR}"/profiles/profiles.desc
 touch -r "${GENTOOX86DIR}"/profiles/profiles.desc "${RSYNCDIR}"/profiles/profiles.desc
+# we will generate thick manifests, so ensure Portage knows that
+sed -i -e '/^thin-manifests/s/true/false/' "${RSYNCDIR}"/metadata/layout.conf
 echo "($(date +"%F %R")) set up repo $(< "${RSYNCDIR}"/profiles/repo_name)"
 
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     abcb776e7f795908ad39f72786b2bb991a5a88b5
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 21 08:50:24 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Feb 21 08:53:25 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=abcb776e

scripts/rsync-generation/update-rsync-master: use CommitDate iso AuthorDate

AuthorDate can be anything, as I just found out, so don't rely on that.
Use CommitDate instead, which is hopefully closer to reality.

 scripts/rsync-generation/update-rsync-master.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index e78b89f896..0a4cf32d15 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -46,7 +46,7 @@ apply_git_mtimes() {
 	local ts=0
 	local files=()
 	{
-		git log --pretty=%at --name-status --reverse "${from}..${to}"
+		git log --pretty=%ct --name-status --reverse "${from}..${to}"
 		echo 999  # end marker to trigger the last block to be done
 	} | \
 	while read line ; do

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     88995923511b937f0a3c6a7218fb071838964c47
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 17 17:17:34 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Feb 17 17:17:34 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=88995923

scripts/rsync-generation/hashgen: update timestamps more sensibly

Directory mtimes don't change when a file gets modified, so don't rely
on it.  Instead, base Manifest mtime on the latest file they describe,
such that they stay the same when nothing changes on a subsequent run.

 scripts/rsync-generation/hashgen.c | 91 ++++++++++++++++++++++++--------------
 1 file changed, 59 insertions(+), 32 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 78944ba9ee..fa0519fb04 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -44,8 +44,23 @@ hex_hash(char *out, const unsigned char *buf, const int length)
 	}
 }
 
+static inline void
+update_times(struct timeval *tv, struct stat *s)
+{
+	if (tv[1].tv_sec < s->st_mtim.tv_sec ||
+			(tv[1].tv_sec == s->st_mtim.tv_sec &&
+			 tv[1].tv_usec < s->st_mtim.tv_nsec / 1000))
+	{
+		tv[0].tv_sec = s->st_atim.tv_sec;
+		tv[0].tv_usec = s->st_atim.tv_nsec / 1000;
+		tv[1].tv_sec = s->st_mtim.tv_sec;
+		tv[1].tv_usec = s->st_mtim.tv_nsec / 1000;
+	}
+}
+
 static void
 write_hashes(
+		struct timeval *tv,
 		const char *root,
 		const char *name,
 		const char *type,
@@ -65,11 +80,15 @@ write_hashes(
 	SHA512_CTX s512;
 	WHIRLPOOL_CTX whrl;
 	blake2b_state bl2b;
+	struct stat s;
 
 	snprintf(fname, sizeof(fname), "%s/%s", root, name);
 	if ((f = fopen(fname, "r")) == NULL)
 		return;
 
+	if (stat(fname, &s) == 0)
+		update_times(tv, &s);
+
 	SHA256_Init(&s256);
 	SHA512_Init(&s512);
 	WHIRLPOOL_Init(&whrl);
@@ -160,7 +179,11 @@ write_hashes(
 }
 
 static char
-write_hashes_dir(const char *root, const char *name, gzFile zm)
+write_hashes_dir(
+		struct timeval *tv,
+		const char *root,
+		const char *name,
+		gzFile zm)
 {
 	char path[8192];
 	DIR *d;
@@ -172,12 +195,11 @@ write_hashes_dir(const char *root, const char *name, gzFile zm)
 			/* skip all dotfiles */
 			if (e->d_name[0] == '.')
 				continue;
-			snprintf(path, sizeof(path), "%s/%s",
-					name, e->d_name);
-			if (write_hashes_dir(root, path, zm))
+			snprintf(path, sizeof(path), "%s/%s", name, e->d_name);
+			if (write_hashes_dir(tv, root, path, zm))
 				continue;
 			/* regular file */
-			write_hashes(root, path, "DATA", NULL, zm);
+			write_hashes(tv, root, path, "DATA", NULL, zm);
 		}
 		closedir(d);
 		return 1;
@@ -192,6 +214,7 @@ process_files(const char *dir, const char *off, FILE *m)
 	char path[8192];
 	DIR *d;
 	struct dirent *e;
+	struct timeval tv[2]; /* dummy, won't use its result */
 
 	snprintf(path, sizeof(path), "%s/%s", dir, off);
 	if ((d = opendir(path)) != NULL) {
@@ -204,7 +227,7 @@ process_files(const char *dir, const char *off, FILE *m)
 			if (process_files(dir, path, m))
 				continue;
 			/* regular file */
-			write_hashes(dir, path, "AUX", m, NULL);
+			write_hashes(tv, dir, path, "AUX", m, NULL);
 		}
 		closedir(d);
 		return 1;
@@ -316,19 +339,18 @@ process_dir(const char *dir)
 	struct stat s;
 	struct timeval tv[2];
 
-	/* set mtime of Manifest(.gz) to the one of the parent dir, this way
-	 * we ensure the Manifest gets mtime bumped upon any change made
-	 * to the directory, that is, a DIST change (Manifest itself) or
-	 * any other change (ebuild, files, metadata) */
-	if (stat(dir, &s)) {
-		tv[0].tv_sec = 0;
-		tv[0].tv_usec = 0;
-	} else {
-		tv[0].tv_sec = s.st_atim.tv_sec;
-		tv[0].tv_usec = s.st_atim.tv_nsec / 1000;
-		tv[1].tv_sec = s.st_mtim.tv_sec;
-		tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
-	}
+	/* our timestamp strategy is as follows:
+	 * - when a Manifest exists, use its timestamp
+	 * - when a meta-Manifest is written (non-ebuilds) use the timestamp
+	 *   of the latest Manifest referenced
+	 * - when a Manifest is written for something like eclasses, use the
+	 *   timestamp of the latest file in the dir
+	 * this way we should keep updates limited to where changes are, and
+	 * also get reproducible mtimes. */
+	tv[0].tv_sec = 0;
+	tv[0].tv_usec = 0;
+	tv[1].tv_sec = 0;
+	tv[1].tv_usec = 0;
 
 	type_manifest = CATEGORY_MANIFEST;
 	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
@@ -366,7 +388,6 @@ process_dir(const char *dir)
 		gzFile mf;
 
 		if ((d = opendir(dir)) != NULL) {
-			struct stat s;
 			char *my_manifest = str_manifest_gz;
 
 			if (type_manifest == GLOBAL_MANIFEST)
@@ -393,7 +414,7 @@ process_dir(const char *dir)
 				if (!stat(path, &s)) {
 					if (s.st_mode & S_IFDIR) {
 						if (type_manifest == SUBTREE_MANIFEST) {
-							write_hashes_dir(dir, e->d_name, mf);
+							write_hashes_dir(tv, dir, e->d_name, mf);
 							if (strcmp(e->d_name, "metadata") == 0) {
 								char buf[2048];
 								size_t len;
@@ -412,10 +433,10 @@ process_dir(const char *dir)
 							}
 							snprintf(path, sizeof(path), "%s/%s",
 									e->d_name, mfest);
-							write_hashes(dir, path, "MANIFEST", NULL, mf);
+							write_hashes(tv, dir, path, "MANIFEST", NULL, mf);
 						}
 					} else if (s.st_mode & S_IFREG) {
-						write_hashes(dir, e->d_name, "DATA", NULL, mf);
+						write_hashes(tv, dir, e->d_name, "DATA", NULL, mf);
 					}
 				}
 			}
@@ -427,6 +448,7 @@ process_dir(const char *dir)
 				size_t len;
 				FILE *m;
 				time_t rtime;
+				struct timeval ntv[2]; /* dummy, not used */
 
 				len = snprintf(buf, sizeof(buf),
 						"IGNORE distfiles\n"
@@ -446,18 +468,13 @@ process_dir(const char *dir)
 					return NULL;
 				}
 
-				write_hashes(dir, my_manifest, "MANIFEST", m, NULL);
+				write_hashes(ntv, dir, my_manifest, "MANIFEST", m, NULL);
 				time(&rtime);
 				len = strftime(buf, sizeof(buf),
 						"TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n", gmtime(&rtime));
 				fwrite(buf, len, 1, m);
 				fflush(m);
 				fclose(m);
-
-				if (tv[0].tv_sec != 0) {
-					/* restore dir mtime, and set Manifest mtime to match it */
-					utimes(globmanifest, tv);
-				}
 			} else {
 				gzclose(mf);
 			}
@@ -511,17 +528,27 @@ process_dir(const char *dir)
 					continue;
 				if (strcmp(e->d_name + strlen(e->d_name) - 7, ".ebuild") != 0)
 					continue;
-				write_hashes(dir, e->d_name, "EBUILD", m, NULL);
+				write_hashes(tv, dir, e->d_name, "EBUILD", m, NULL);
 			}
 			closedir(d);
 		}
 
-		write_hashes(dir, "ChangeLog", "MISC", m, NULL);
-		write_hashes(dir, "metadata.xml", "MISC", m, NULL);
+		write_hashes(tv, dir, "ChangeLog", "MISC", m, NULL);
+		write_hashes(tv, dir, "metadata.xml", "MISC", m, NULL);
 
 		fflush(m);
 		fclose(m);
 
+		if (stat(manifest, &s)) {
+			tv[0].tv_sec = 0;
+			tv[0].tv_usec = 0;
+		} else {
+			tv[0].tv_sec = s.st_atim.tv_sec;
+			tv[0].tv_usec = s.st_atim.tv_nsec / 1000;
+			tv[1].tv_sec = s.st_mtim.tv_sec;
+			tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
+		}
+
 		rename(newmanifest, manifest);
 		if (tv[0].tv_sec != 0) {
 			/* restore dir mtime, and set Manifest mtime to match it */

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     7555f191b3b49230fa00d9bb85da5c2e56928189
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 17 08:11:44 2018 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Feb 17 08:11:44 2018 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=7555f191

scripts/rsync-generation/update-rsync-master: generate/sign manifest last

Generate manifests as last thing, else we invalidate the signatures due
to other work preparing the tree.

 scripts/rsync-generation/update-rsync-master.sh | 56 ++++++++++++-------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 459edebce0..e78b89f896 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -185,34 +185,6 @@ echo "($(date +"%F %R")) Prefix tree rsynced"
 STOP=$(date +%s)
 TIME_SVNPREFIX=$((STOP - START))
 
-START=$(date +%s)
-
-echo "($(date +"%F %R")) signing Manifest"
-
-# generate Thick Manifests
-${BASE_PATH}/hashgen "${RSYNCDIR}"
-
-# Signing is done with our snapshot signing key, and only on the top
-# level Manifest, for it covers indirectly the entire tree
-
-# remember, HOME is set to misc/ so .gnupg keychain lives there
-gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
-	--pinentry-mode loopback \
-	--sign --clearsign --digest-algo SHA512 \
-	--yes "${RSYNCDIR}"/Manifest \
-	< "${BASE_PATH}"/autosigner.pwd >& /dev/null
-if [[ -f ${RSYNCDIR}/Manifest.asc ]] ; then
-	touch -r "${RSYNCDIR}"/Manifest "${RSYNCDIR}"/Manifest.asc
-	mv "${RSYNCDIR}"/Manifest{.asc,}
-else
-	echo "signing failed!" >> /dev/stderr
-fi
-
-echo "($(date +"%F %R")) Manifest signed"
-
-STOP=$(date +%s)
-TIME_MANISIGN=$((STOP - START))
-
 
 # define repo_name, can't use gx86's name as we're different
 echo "($(date +"%F %R")) setting repo_name and making the prefix profiles development ones (iso exp)"
@@ -264,6 +236,34 @@ chmod -R u-s,g-s "${RSYNCDIR}"/metadata
 STOP=$(date +%s)
 TIME_TOTAL=$((STOP - GLOBALSTART))
 
+START=$(date +%s)
+
+echo "($(date +"%F %R")) signing Manifest"
+
+# generate Thick Manifests
+${BASE_PATH}/hashgen "${RSYNCDIR}"
+
+# Signing is done with our snapshot signing key, and only on the top
+# level Manifest, for it covers indirectly the entire tree
+
+# remember, HOME is set to misc/ so .gnupg keychain lives there
+gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
+	--pinentry-mode loopback \
+	--sign --clearsign --digest-algo SHA512 \
+	--yes "${RSYNCDIR}"/Manifest \
+	< "${BASE_PATH}"/autosigner.pwd 2>&1
+if [[ -f ${RSYNCDIR}/Manifest.asc ]] ; then
+	touch -r "${RSYNCDIR}"/Manifest "${RSYNCDIR}"/Manifest.asc
+	mv "${RSYNCDIR}"/Manifest{.asc,}
+else
+	echo "signing failed!" >> /dev/stderr
+fi
+
+echo "($(date +"%F %R")) Manifest signed"
+
+STOP=$(date +%s)
+TIME_MANISIGN=$((STOP - START))
+
 # feed timings to graphite
 prefix="gentoo.rsync-generation.$(hostname -s)"
 {

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     ec87cd2abce74fe8378b7c9aa9b89cb2ccc5bff7
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Dec  1 13:42:20 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Dec  1 13:42:20 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=ec87cd2a

hashgen: don't generate Manifest.gz for all subdirs

It turns out the generator can decide where Manifest files are created
(e.g. there is no spec).  Because gemato is not generating Manifest
files in some subdirs (most notably install-qa-check.d) apply a simpler
strategy for this, just excluding dirs like metadata from Manifest files
in each and every subdir.

 scripts/rsync-generation/hashgen.c | 105 +++++++++++++++++++++++++++++++------
 1 file changed, 90 insertions(+), 15 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 0a00c215f7..78944ba9ee 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -160,6 +160,33 @@ write_hashes(
 }
 
 static char
+write_hashes_dir(const char *root, const char *name, gzFile zm)
+{
+	char path[8192];
+	DIR *d;
+	struct dirent *e;
+
+	snprintf(path, sizeof(path), "%s/%s", root, name);
+	if ((d = opendir(path)) != NULL) {
+		while ((e = readdir(d)) != NULL) {
+			/* skip all dotfiles */
+			if (e->d_name[0] == '.')
+				continue;
+			snprintf(path, sizeof(path), "%s/%s",
+					name, e->d_name);
+			if (write_hashes_dir(root, path, zm))
+				continue;
+			/* regular file */
+			write_hashes(root, path, "DATA", NULL, zm);
+		}
+		closedir(d);
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+static char
 process_files(const char *dir, const char *off, FILE *m)
 {
 	char path[8192];
@@ -278,8 +305,14 @@ process_dir(const char *dir)
 	DIR *d;
 	struct dirent *e;
 	char path[8192];
+	const char *p;
 	int newhashes;
-	char global_manifest = 0;
+	enum {
+		GLOBAL_MANIFEST,   /* Manifest.files.gz + Manifest */
+		SUBTREE_MANIFEST,  /* Manifest.gz for recursive list of files */
+		EBUILD_MANIFEST,   /* Manifest thick from thin */
+		CATEGORY_MANIFEST  /* Manifest.gz with Manifest entries */
+	} type_manifest;
 	struct stat s;
 	struct timeval tv[2];
 
@@ -297,24 +330,48 @@ process_dir(const char *dir)
 		tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
 	}
 
+	type_manifest = CATEGORY_MANIFEST;
 	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
 	if ((newhashes = parse_layout_conf(path)) != 0) {
-		global_manifest = 1;
+		type_manifest = GLOBAL_MANIFEST;
 		hashes = newhashes;
+	} else {
+		if ((p = strrchr(dir, '/')) != NULL) {
+			p++;
+		} else {
+			p = dir;
+		}
+
+		if (
+				strcmp(p, "eclass") == 0 ||
+				strcmp(p, "licenses") == 0 ||
+				strcmp(p, "metadata") == 0 ||
+				strcmp(p, "profiles") == 0 ||
+				strcmp(p, "scripts") == 0
+			)
+		{
+			type_manifest = SUBTREE_MANIFEST;
+		}
 	}
 
+	/* If a Manifest file exists, this is an ebuild dir, unless we
+	 * already established this is the top level dir which also has a
+	 * Manifest file. */
 	snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest);
-	if (global_manifest || (f = fopen(manifest, "r")) == NULL) {
+	if (type_manifest == GLOBAL_MANIFEST ||
+			(f = fopen(manifest, "r")) == NULL)
+	{
+		/* all of these types (GLOBAL, SUBTREE, CATEGORY) have a gzipped
+		 * Manifest */
 		gzFile mf;
 
-		/* recurse into subdirs */
 		if ((d = opendir(dir)) != NULL) {
 			struct stat s;
-			char *my_manifest =
-				global_manifest ? str_manifest_files_gz : str_manifest_gz;
+			char *my_manifest = str_manifest_gz;
+
+			if (type_manifest == GLOBAL_MANIFEST)
+				my_manifest = str_manifest_files_gz;
 
-			/* open up a gzipped Manifest to keep the hashes of the
-			 * Manifests in the subdirs */
 			snprintf(manifest, sizeof(manifest), "%s/%s", dir, my_manifest);
 			if ((mf = gzopen(manifest, "wb9")) == NULL) {
 				fprintf(stderr, "failed to open file '%s' for writing: %s\n",
@@ -323,22 +380,40 @@ process_dir(const char *dir)
 			}
 
 			while ((e = readdir(d)) != NULL) {
+				/* ignore dotfiles (including . and ..) */
 				if (e->d_name[0] == '.')
 					continue;
+				/* ignore existing Manifests */
 				if (strcmp(e->d_name, my_manifest) == 0)
 					continue;
 				if (strcmp(e->d_name, str_manifest) == 0)
 					continue;
+
 				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
 				if (!stat(path, &s)) {
 					if (s.st_mode & S_IFDIR) {
-						char *mfest = process_dir(path);
-						if (mfest == NULL) {
-							gzclose(mf);
-							return NULL;
+						if (type_manifest == SUBTREE_MANIFEST) {
+							write_hashes_dir(dir, e->d_name, mf);
+							if (strcmp(e->d_name, "metadata") == 0) {
+								char buf[2048];
+								size_t len;
+								len = snprintf(buf, sizeof(buf),
+										"IGNORE timestamp\n"
+										"IGNORE timestamp.chk\n"
+										"IGNORE timestamp.commit\n"
+										"IGNORE timestamp.x\n");
+								gzwrite(mf, buf, len);
+							}
+						} else {
+							char *mfest = process_dir(path);
+							if (mfest == NULL) {
+								gzclose(mf);
+								return NULL;
+							}
+							snprintf(path, sizeof(path), "%s/%s",
+									e->d_name, mfest);
+							write_hashes(dir, path, "MANIFEST", NULL, mf);
 						}
-						snprintf(path, sizeof(path), "%s/%s", e->d_name, mfest);
-						write_hashes(dir, path, "MANIFEST", NULL, mf);
 					} else if (s.st_mode & S_IFREG) {
 						write_hashes(dir, e->d_name, "DATA", NULL, mf);
 					}
@@ -346,7 +421,7 @@ process_dir(const char *dir)
 			}
 			closedir(d);
 
-			if (global_manifest) {
+			if (type_manifest == GLOBAL_MANIFEST) {
 				char globmanifest[8192];
 				char buf[2048];
 				size_t len;

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     514450bb4e8e687f882f1a2f1cb1e302880533d4
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 21:33:13 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 21:33:13 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=514450bb

hashgen: ignore Manifest in top level Manifest.files.gz

 scripts/rsync-generation/hashgen.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 79d26a9ae7..0a00c215f7 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -327,6 +327,8 @@ process_dir(const char *dir)
 					continue;
 				if (strcmp(e->d_name, my_manifest) == 0)
 					continue;
+				if (strcmp(e->d_name, str_manifest) == 0)
+					continue;
 				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
 				if (!stat(path, &s)) {
 					if (s.st_mode & S_IFDIR) {

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     822d7ca7cb015470e7805f9888ba5255309819ba
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 21:31:26 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 21:31:26 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=822d7ca7

hashgen: improve layout.conf parsing, return default hashes if found

 scripts/rsync-generation/hashgen.c | 75 +++++++++++++++++++++-----------------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 25ae1db70e..79d26a9ae7 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -32,7 +32,8 @@ enum hash_impls {
 };
 /* default changed from sha256, sha512, whirlpool
  * to blake2b, sha512 on 2017-11-21 */
-static int hashes = HASH_BLAKE2B | HASH_SHA512;
+#define HASH_DEFAULT  (HASH_BLAKE2B | HASH_SHA512);
+static int hashes = HASH_DEFAULT;
 
 static inline void
 hex_hash(char *out, const unsigned char *buf, const int length)
@@ -196,6 +197,7 @@ parse_layout_conf(const char *path)
 	char *q;
 	char *tok;
 	char *last_nl;
+	char *start;
 	int ret = 0;
 
 	if ((f = fopen(path, "r")) == NULL)
@@ -205,57 +207,64 @@ parse_layout_conf(const char *path)
 	 * if the file doesn't end with a newline, the final bit is ignored */
 	while ((sz = fread(buf + len, 1, sizeof(buf) - len, f)) > 0) {
 		len += sz;
+		start = buf;
 		last_nl = NULL;
 		for (p = buf; p - buf < len; p++) {
 			if (*p == '\n') {
+				if (last_nl != NULL)
+					start = last_nl + 1;
 				last_nl = p;
-				sz = strlen("manifest-hashes");
-				if (strncmp(buf, "manifest-hashes", sz))
-					continue;
-				if ((q = strchr(buf + sz, '=')) == NULL)
-					continue;
-				q++;
-				while (isspace((int)*q))
-					q++;
-				/* parse the tokens, whitespace separated */
-				tok = q;
 				do {
-					while (!isspace((int)*q))
-						q++;
-					sz = q - tok;
-					if (strncmp(tok, "SHA256", sz) == 0) {
-						ret |= HASH_SHA256;
-					} else if (strncmp(tok, "SHA512", sz) == 0) {
-						ret |= HASH_SHA512;
-					} else if (strncmp(tok, "WHIRLPOOL", sz) == 0) {
-						ret |= HASH_WHIRLPOOL;
-					} else if (strncmp(tok, "BLAKE2B", sz) == 0) {
-						ret |= HASH_BLAKE2B;
-					} else {
-						fprintf(stderr, "warning: unsupported hash from "
-								"layout.conf: %.*s\n", (int)sz, tok);
-					}
-					while (isspace((int)*q) && *q != '\n')
+					sz = strlen("manifest-hashes");
+					if (strncmp(start, "manifest-hashes", sz))
+						break;
+					if ((q = strchr(start + sz, '=')) == NULL)
+						break;
+					q++;
+					while (isspace((int)*q))
 						q++;
+					/* parse the tokens, whitespace separated */
 					tok = q;
-				} while (*q != '\n');
-				/* got it, expect only once, so stop processing */
-				fclose(f);
-				return ret;
+					do {
+						while (!isspace((int)*q))
+							q++;
+						sz = q - tok;
+						if (strncmp(tok, "SHA256", sz) == 0) {
+							ret |= HASH_SHA256;
+						} else if (strncmp(tok, "SHA512", sz) == 0) {
+							ret |= HASH_SHA512;
+						} else if (strncmp(tok, "WHIRLPOOL", sz) == 0) {
+							ret |= HASH_WHIRLPOOL;
+						} else if (strncmp(tok, "BLAKE2B", sz) == 0) {
+							ret |= HASH_BLAKE2B;
+						} else {
+							fprintf(stderr, "warning: unsupported hash from "
+									"layout.conf: %.*s\n", (int)sz, tok);
+						}
+						while (isspace((int)*q) && *q != '\n')
+							q++;
+						tok = q;
+					} while (*q != '\n');
+					/* got it, expect only once, so stop processing */
+					fclose(f);
+					return ret;
+				} while (0);
 			}
 		}
 		if (last_nl != NULL) {
 			last_nl++;  /* skip \n */
 			len = last_nl - buf;
 			memmove(buf, last_nl, len);
+			last_nl = buf;
 		} else {
-			/* too long line, just skip */
+			/* skip too long line */
 			len = 0;
 		}
 	}
 
 	fclose(f);
-	return 0;
+	/* if we didn't find anything, return the default set */
+	return HASH_DEFAULT;
 }
 
 static char *str_manifest = "Manifest";

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     8df1e3c218b317e9e780d081663a8dbee0ac817e
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 18:56:55 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 18:56:55 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=8df1e3c2

hashgen: ensure top level Manifest isn't taken for package

 scripts/rsync-generation/hashgen.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 4180665e0f..7700e88198 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -292,7 +292,7 @@ process_dir(const char *dir)
 	}
 
 	snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest);
-	if ((f = fopen(manifest, "r")) == NULL) {
+	if (global_manifest || (f = fopen(manifest, "r")) == NULL) {
 		gzFile mf;
 
 		/* recurse into subdirs */

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f931cecd430f96ab3ae7c3b1561104c7561af387
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 19:00:03 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 19:00:03 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f931cecd

hashgen: warn when an unsupported hash is found

 scripts/rsync-generation/hashgen.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 7700e88198..25ae1db70e 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -231,6 +231,9 @@ parse_layout_conf(const char *path)
 						ret |= HASH_WHIRLPOOL;
 					} else if (strncmp(tok, "BLAKE2B", sz) == 0) {
 						ret |= HASH_BLAKE2B;
+					} else {
+						fprintf(stderr, "warning: unsupported hash from "
+								"layout.conf: %.*s\n", (int)sz, tok);
 					}
 					while (isspace((int)*q) && *q != '\n')
 						q++;

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     17ac45552c0f6c49f28e11fad23ab2cddfdd5393
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 19:26:38 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 19:26:38 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=17ac4555

update-rsync-master: only sign the top level Manifest

 scripts/rsync-generation/update-rsync-master.sh | 69 +++++++------------------
 1 file changed, 20 insertions(+), 49 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 5f73206eae..459edebce0 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -187,57 +187,28 @@ TIME_SVNPREFIX=$((STOP - START))
 
 START=$(date +%s)
 
-echo "($(date +"%F %R")) signing unsigned Manifests"
+echo "($(date +"%F %R")) signing Manifest"
 
 # generate Thick Manifests
-${BASE_PATH}/hashgen ${RSYNCDIR}
-
-# We store signed Manifests in a "cache", so we don't have to
-# generate them all-over all the time.  Generation needs to take place
-# if:
-# 1. the original Manifest isn't signed
-# 2. we don't have one generated file
-# 3. the Manifest modification time is newer than our generated file
-# Signing is done with our snapshot signing key
-sign_manifest() {
-	local pkg=$1
-	local mc=${pkg//\//_}.manifest
-	[[ -z ${pkg} ]] && return 1
-
-	if [[ ! -f ${MANIFEST_CACHE}/${mc} || ${RSYNCDIR}/${pkg}/Manifest -nt ${MANIFEST_CACHE}/${mc} ]] ; then
-		mkdir -p "${MANIFEST_CACHE}"
-
-		echo "Signing Manifest for ${pkg}"
-		cat "${RSYNCDIR}/${pkg}"/Manifest > "${MANIFEST_CACHE}"/${mc}
-		# remember, HOME is set to misc/ so .gnupg keychain lives there
-		gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
-			--pinentry-mode loopback \
-			--sign --clearsign --digest-algo SHA512 \
-			--yes "${MANIFEST_CACHE}"/${mc} \
-			< "${BASE_PATH}"/autosigner.pwd >& /dev/null
-		if [[ -f ${MANIFEST_CACHE}/${mc}.asc ]] ; then
-			touch -r "${RSYNCDIR}/${pkg}"/Manifest \
-				"${MANIFEST_CACHE}"/${mc}.asc
-			mv "${MANIFEST_CACHE}"/${mc}{.asc,}
-		else
-			rm "${MANIFEST_CACHE}"/${mc}
-			echo "signing failed!" >> /dev/stderr
-			return 0
-		fi
-	fi
-
-	cp -a "${MANIFEST_CACHE}"/${mc} "${RSYNCDIR}/${pkg}"/Manifest
-
-	return 0
-}
-
-for entry in "${RSYNCDIR}"/*/* ; do
-	[[ ! -f "${entry}"/Manifest ]] && continue
-	entry=${entry#${RSYNCDIR}/}
-	sign_manifest "${entry}"
-done
-
-echo "($(date +"%F %R")) unsigned Manifests signed"
+${BASE_PATH}/hashgen "${RSYNCDIR}"
+
+# Signing is done with our snapshot signing key, and only on the top
+# level Manifest, for it covers indirectly the entire tree
+
+# remember, HOME is set to misc/ so .gnupg keychain lives there
+gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
+	--pinentry-mode loopback \
+	--sign --clearsign --digest-algo SHA512 \
+	--yes "${RSYNCDIR}"/Manifest \
+	< "${BASE_PATH}"/autosigner.pwd >& /dev/null
+if [[ -f ${RSYNCDIR}/Manifest.asc ]] ; then
+	touch -r "${RSYNCDIR}"/Manifest "${RSYNCDIR}"/Manifest.asc
+	mv "${RSYNCDIR}"/Manifest{.asc,}
+else
+	echo "signing failed!" >> /dev/stderr
+fi
+
+echo "($(date +"%F %R")) Manifest signed"
 
 STOP=$(date +%s)
 TIME_MANISIGN=$((STOP - START))

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     191df7d69bc652b5e7899d44934fea0de05aa53d
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 16:42:54 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 16:42:54 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=191df7d6

hashgen: generate GLEP 74 top level Manifest

 scripts/rsync-generation/hashgen.c | 56 ++++++++++++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 034ea7f170..4180665e0f 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -4,6 +4,7 @@
 #include <strings.h>
 #include <ctype.h>
 #include <dirent.h>
+#include <time.h>
 #include <errno.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -95,7 +96,7 @@ write_hashes(
 #pragma omp section
 			{
 				if (hashes & HASH_BLAKE2B)
-					blake2b_update(&bl2b, data, len);
+					blake2b_update(&bl2b, (unsigned char *)data, len);
 			}
 		}
 	}
@@ -152,7 +153,7 @@ write_hashes(
 	len += snprintf(data + len, sizeof(data) - len, "\n");
 
 	if (m != NULL)
-		fwrite(data, 1, len, m);
+		fwrite(data, len, 1, m);
 	if (gm != NULL)
 		gzwrite(gm, data, len);
 }
@@ -202,7 +203,7 @@ parse_layout_conf(const char *path)
 
 	/* read file, examine lines after encountering a newline, that is,
 	 * if the file doesn't end with a newline, the final bit is ignored */
-	while (sz = fread(buf + len, 1, sizeof(buf) - len, f) > 0) {
+	while ((sz = fread(buf + len, 1, sizeof(buf) - len, f)) > 0) {
 		len += sz;
 		last_nl = NULL;
 		for (p = buf; p - buf < len; p++) {
@@ -256,6 +257,7 @@ parse_layout_conf(const char *path)
 
 static char *str_manifest = "Manifest";
 static char *str_manifest_gz = "Manifest.gz";
+static char *str_manifest_files_gz = "Manifest.files.gz";
 static char *
 process_dir(const char *dir)
 {
@@ -296,10 +298,12 @@ process_dir(const char *dir)
 		/* recurse into subdirs */
 		if ((d = opendir(dir)) != NULL) {
 			struct stat s;
+			char *my_manifest =
+				global_manifest ? str_manifest_files_gz : str_manifest_gz;
 
 			/* open up a gzipped Manifest to keep the hashes of the
 			 * Manifests in the subdirs */
-			snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest_gz);
+			snprintf(manifest, sizeof(manifest), "%s/%s", dir, my_manifest);
 			if ((mf = gzopen(manifest, "wb9")) == NULL) {
 				fprintf(stderr, "failed to open file '%s' for writing: %s\n",
 						manifest, strerror(errno));
@@ -309,7 +313,7 @@ process_dir(const char *dir)
 			while ((e = readdir(d)) != NULL) {
 				if (e->d_name[0] == '.')
 					continue;
-				if (strcmp(e->d_name, str_manifest_gz) == 0)
+				if (strcmp(e->d_name, my_manifest) == 0)
 					continue;
 				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
 				if (!stat(path, &s)) {
@@ -328,7 +332,47 @@ process_dir(const char *dir)
 			}
 			closedir(d);
 
-			gzclose(mf);
+			if (global_manifest) {
+				char globmanifest[8192];
+				char buf[2048];
+				size_t len;
+				FILE *m;
+				time_t rtime;
+
+				len = snprintf(buf, sizeof(buf),
+						"IGNORE distfiles\n"
+						"IGNORE local\n"
+						"IGNORE lost+found\n"
+						"IGNORE packages\n");
+				gzwrite(mf, buf, len);
+				gzclose(mf);
+
+				/* create global Manifest */
+				snprintf(globmanifest, sizeof(globmanifest),
+						"%s/%s", dir, str_manifest);
+				if ((m = fopen(globmanifest, "w")) == NULL) {
+					fprintf(stderr, "failed to open file '%s' "
+							"for writing: %s\n",
+							globmanifest, strerror(errno));
+					return NULL;
+				}
+
+				write_hashes(dir, my_manifest, "MANIFEST", m, NULL);
+				time(&rtime);
+				len = strftime(buf, sizeof(buf),
+						"TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n", gmtime(&rtime));
+				fwrite(buf, len, 1, m);
+				fflush(m);
+				fclose(m);
+
+				if (tv[0].tv_sec != 0) {
+					/* restore dir mtime, and set Manifest mtime to match it */
+					utimes(globmanifest, tv);
+				}
+			} else {
+				gzclose(mf);
+			}
+
 			if (tv[0].tv_sec != 0) {
 				/* restore dir mtime, and set Manifest mtime to match it */
 				utimes(manifest, tv);

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     8134627701e2eea2a0eb590f6bd80064a2496ecf
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 15:35:43 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 15:35:43 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=81346277

hashgen: avoid empty Manifest references

 scripts/rsync-generation/hashgen.c | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 68d5575ad2..034ea7f170 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -289,25 +289,28 @@ process_dir(const char *dir)
 		hashes = newhashes;
 	}
 
-	snprintf(manifest, sizeof(manifest), "%s/Manifest", dir);
+	snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest);
 	if ((f = fopen(manifest, "r")) == NULL) {
 		gzFile mf;
 
-		/* open up a gzipped Manifest to keep the hashes of the
-		 * Manifests in the subdirs */
-		snprintf(manifest, sizeof(manifest), "%s/Manifest.gz", dir);
-		if ((mf = gzopen(manifest, "wb9")) == NULL) {
-			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
-					manifest, strerror(errno));
-			return NULL;
-		}
-
 		/* recurse into subdirs */
 		if ((d = opendir(dir)) != NULL) {
 			struct stat s;
+
+			/* open up a gzipped Manifest to keep the hashes of the
+			 * Manifests in the subdirs */
+			snprintf(manifest, sizeof(manifest), "%s/%s", dir, str_manifest_gz);
+			if ((mf = gzopen(manifest, "wb9")) == NULL) {
+				fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+						manifest, strerror(errno));
+				return NULL;
+			}
+
 			while ((e = readdir(d)) != NULL) {
 				if (e->d_name[0] == '.')
 					continue;
+				if (strcmp(e->d_name, str_manifest_gz) == 0)
+					continue;
 				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
 				if (!stat(path, &s)) {
 					if (s.st_mode & S_IFDIR) {
@@ -324,13 +327,13 @@ process_dir(const char *dir)
 				}
 			}
 			closedir(d);
-		}
 
-		gzclose(mf);
-		if (tv[0].tv_sec != 0) {
-			/* restore dir mtime, and set Manifest mtime to match it */
-			utimes(manifest, tv);
-			utimes(dir, tv);
+			gzclose(mf);
+			if (tv[0].tv_sec != 0) {
+				/* restore dir mtime, and set Manifest mtime to match it */
+				utimes(manifest, tv);
+				utimes(dir, tv);
+			}
 		}
 
 		return str_manifest_gz;

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     bb8b83447563ef869f083703e01ad9d6fdceb1cb
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 29 14:28:01 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Nov 29 14:29:00 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=bb8b8344

hashgen: first pass on generating cascading Manifest.gz files

 scripts/rsync-generation/hashgen.c | 215 +++++++++++++++++++++++++++++--------
 1 file changed, 173 insertions(+), 42 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index ddc52752b4..68d5575ad2 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <strings.h>
+#include <ctype.h>
 #include <dirent.h>
 #include <errno.h>
 #include <sys/stat.h>
@@ -10,14 +11,16 @@
 #include <openssl/sha.h>
 #include <openssl/whrlpool.h>
 #include <blake2.h>
+#include <zlib.h>
 
 /* Generate thick Manifests based on thin Manifests */
 
 /* In order to build this program, the following packages are required:
  * - app-crypt/libb2 (for BLAKE2, for as long as openssl doesn't include it)
  * - dev-libs/openssl (for SHA, WHIRLPOOL)
+ * - sys-libs/zlib (for compressing Manifest files)
  * compile like this
- *   ${CC} -o hashgen -fopenmp ${CFLAGS} -lssl -lcrypto -lb2 hashgen.c
+ *   ${CC} -o hashgen -fopenmp ${CFLAGS} -lssl -lcrypto -lb2 -lz hashgen.c
  */
 
 enum hash_impls {
@@ -26,7 +29,8 @@ enum hash_impls {
 	HASH_WHIRLPOOL = 1<<2,
 	HASH_BLAKE2B   = 1<<3
 };
-/* default changed from sha256, sha512, whirlpool to blake2b, sha512 */
+/* default changed from sha256, sha512, whirlpool
+ * to blake2b, sha512 on 2017-11-21 */
 static int hashes = HASH_BLAKE2B | HASH_SHA512;
 
 static inline void
@@ -39,16 +43,21 @@ hex_hash(char *out, const unsigned char *buf, const int length)
 }
 
 static void
-write_hashes(const char *root, const char *name, const char *type, FILE *m)
+write_hashes(
+		const char *root,
+		const char *name,
+		const char *type,
+		FILE *m,
+		gzFile gm)
 {
 	FILE *f;
-	char fname[8096];
+	char fname[8192];
 	size_t flen = 0;
 	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
 	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
 	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
 	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
-	char data[8096];
+	char data[8192];
 	size_t len;
 	SHA256_CTX s256;
 	SHA512_CTX s512;
@@ -121,28 +130,37 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 			if (hashes & HASH_BLAKE2B) {
 				unsigned char blak2bbuf[BLAKE2B_OUTBYTES];
 				blake2b_final(&bl2b, blak2bbuf, BLAKE2B_OUTBYTES);
-				hex_hash(blak2b, blak2bbuf, WHIRLPOOL_DIGEST_LENGTH);
+				hex_hash(blak2b, blak2bbuf, BLAKE2B_OUTBYTES);
 			}
 		}
 	}
 	fclose(f);
 
-	fprintf(m, "%s %s %zd",type, name, flen);
+	len = snprintf(data, sizeof(data), "%s %s %zd", type, name, flen);
 	if (hashes & HASH_BLAKE2B)
-		fprintf(m, " BLAKE2B %s", blak2b);
+		len += snprintf(data + len, sizeof(data) - len,
+				" BLAKE2B %s", blak2b);
 	if (hashes & HASH_SHA256)
-		fprintf(m, " SHA256 %s", sha256);
+		len += snprintf(data + len, sizeof(data) - len,
+				" SHA256 %s", sha256);
 	if (hashes & HASH_SHA512)
-		fprintf(m, " SHA512 %s", sha512);
+		len += snprintf(data + len, sizeof(data) - len,
+				" SHA512 %s", sha512);
 	if (hashes & HASH_WHIRLPOOL)
-		fprintf(m, " WHIRLPOOL %s", whrlpl);
-	fprintf(m, "\n");
+		len += snprintf(data + len, sizeof(data) - len,
+				" WHIRLPOOL %s", whrlpl);
+	len += snprintf(data + len, sizeof(data) - len, "\n");
+
+	if (m != NULL)
+		fwrite(data, 1, len, m);
+	if (gm != NULL)
+		gzwrite(gm, data, len);
 }
 
 static char
 process_files(const char *dir, const char *off, FILE *m)
 {
-	char path[8096];
+	char path[8192];
 	DIR *d;
 	struct dirent *e;
 
@@ -157,7 +175,7 @@ process_files(const char *dir, const char *off, FILE *m)
 			if (process_files(dir, path, m))
 				continue;
 			/* regular file */
-			write_hashes(dir, path, "AUX", m);
+			write_hashes(dir, path, "AUX", m, NULL);
 		}
 		closedir(d);
 		return 1;
@@ -166,17 +184,124 @@ process_files(const char *dir, const char *off, FILE *m)
 	}
 }
 
-static void
+static int
+parse_layout_conf(const char *path)
+{
+	FILE *f;
+	char buf[8192];
+	size_t len = 0;
+	size_t sz;
+	char *p;
+	char *q;
+	char *tok;
+	char *last_nl;
+	int ret = 0;
+
+	if ((f = fopen(path, "r")) == NULL)
+		return 0;
+
+	/* read file, examine lines after encountering a newline, that is,
+	 * if the file doesn't end with a newline, the final bit is ignored */
+	while (sz = fread(buf + len, 1, sizeof(buf) - len, f) > 0) {
+		len += sz;
+		last_nl = NULL;
+		for (p = buf; p - buf < len; p++) {
+			if (*p == '\n') {
+				last_nl = p;
+				sz = strlen("manifest-hashes");
+				if (strncmp(buf, "manifest-hashes", sz))
+					continue;
+				if ((q = strchr(buf + sz, '=')) == NULL)
+					continue;
+				q++;
+				while (isspace((int)*q))
+					q++;
+				/* parse the tokens, whitespace separated */
+				tok = q;
+				do {
+					while (!isspace((int)*q))
+						q++;
+					sz = q - tok;
+					if (strncmp(tok, "SHA256", sz) == 0) {
+						ret |= HASH_SHA256;
+					} else if (strncmp(tok, "SHA512", sz) == 0) {
+						ret |= HASH_SHA512;
+					} else if (strncmp(tok, "WHIRLPOOL", sz) == 0) {
+						ret |= HASH_WHIRLPOOL;
+					} else if (strncmp(tok, "BLAKE2B", sz) == 0) {
+						ret |= HASH_BLAKE2B;
+					}
+					while (isspace((int)*q) && *q != '\n')
+						q++;
+					tok = q;
+				} while (*q != '\n');
+				/* got it, expect only once, so stop processing */
+				fclose(f);
+				return ret;
+			}
+		}
+		if (last_nl != NULL) {
+			last_nl++;  /* skip \n */
+			len = last_nl - buf;
+			memmove(buf, last_nl, len);
+		} else {
+			/* too long line, just skip */
+			len = 0;
+		}
+	}
+
+	fclose(f);
+	return 0;
+}
+
+static char *str_manifest = "Manifest";
+static char *str_manifest_gz = "Manifest.gz";
+static char *
 process_dir(const char *dir)
 {
-	char manifest[8096];
+	char manifest[8192];
 	FILE *f;
 	DIR *d;
 	struct dirent *e;
-	char path[8096];
+	char path[8192];
+	int newhashes;
+	char global_manifest = 0;
+	struct stat s;
+	struct timeval tv[2];
+
+	/* set mtime of Manifest(.gz) to the one of the parent dir, this way
+	 * we ensure the Manifest gets mtime bumped upon any change made
+	 * to the directory, that is, a DIST change (Manifest itself) or
+	 * any other change (ebuild, files, metadata) */
+	if (stat(dir, &s)) {
+		tv[0].tv_sec = 0;
+		tv[0].tv_usec = 0;
+	} else {
+		tv[0].tv_sec = s.st_atim.tv_sec;
+		tv[0].tv_usec = s.st_atim.tv_nsec / 1000;
+		tv[1].tv_sec = s.st_mtim.tv_sec;
+		tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
+	}
+
+	snprintf(path, sizeof(path), "%s/metadata/layout.conf", dir);
+	if ((newhashes = parse_layout_conf(path)) != 0) {
+		global_manifest = 1;
+		hashes = newhashes;
+	}
 
 	snprintf(manifest, sizeof(manifest), "%s/Manifest", dir);
 	if ((f = fopen(manifest, "r")) == NULL) {
+		gzFile mf;
+
+		/* open up a gzipped Manifest to keep the hashes of the
+		 * Manifests in the subdirs */
+		snprintf(manifest, sizeof(manifest), "%s/Manifest.gz", dir);
+		if ((mf = gzopen(manifest, "wb9")) == NULL) {
+			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
+					manifest, strerror(errno));
+			return NULL;
+		}
+
 		/* recurse into subdirs */
 		if ((d = opendir(dir)) != NULL) {
 			struct stat s;
@@ -184,38 +309,42 @@ process_dir(const char *dir)
 				if (e->d_name[0] == '.')
 					continue;
 				snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
-				if (!stat(path, &s) && s.st_mode & S_IFDIR)
-					process_dir(path);
+				if (!stat(path, &s)) {
+					if (s.st_mode & S_IFDIR) {
+						char *mfest = process_dir(path);
+						if (mfest == NULL) {
+							gzclose(mf);
+							return NULL;
+						}
+						snprintf(path, sizeof(path), "%s/%s", e->d_name, mfest);
+						write_hashes(dir, path, "MANIFEST", NULL, mf);
+					} else if (s.st_mode & S_IFREG) {
+						write_hashes(dir, e->d_name, "DATA", NULL, mf);
+					}
+				}
 			}
 			closedir(d);
 		}
+
+		gzclose(mf);
+		if (tv[0].tv_sec != 0) {
+			/* restore dir mtime, and set Manifest mtime to match it */
+			utimes(manifest, tv);
+			utimes(dir, tv);
+		}
+
+		return str_manifest_gz;
 	} else {
 		/* this looks like an ebuild dir, so update the Manifest */
 		FILE *m;
-		char newmanifest[8096];
-		char buf[8096];
-		struct stat s;
-		struct timeval tv[2];
-
-		/* set mtime of Manifest to the one of the parent dir, this way
-		 * we ensure the Manifest gets mtime bumped upon any change made
-		 * to the directory, that is, a DIST change (Manifest itself) or
-		 * any other change (ebuild, files, metadata) */
-		if (stat(dir, &s)) {
-			tv[0].tv_sec = 0;
-			tv[0].tv_usec = 0;
-		} else {
-			tv[0].tv_sec = s.st_atim.tv_sec;
-			tv[0].tv_usec = s.st_atim.tv_nsec / 1000;
-			tv[1].tv_sec = s.st_mtim.tv_sec;
-			tv[1].tv_usec = s.st_mtim.tv_nsec / 1000;
-		}
+		char newmanifest[8192];
+		char buf[8192];
 
 		snprintf(newmanifest, sizeof(newmanifest), "%s/.Manifest.new", dir);
 		if ((m = fopen(newmanifest, "w")) == NULL) {
 			fprintf(stderr, "failed to open file '%s' for writing: %s\n",
 					newmanifest, strerror(errno));
-			return;
+			return NULL;
 		}
 
 		/* we know the Manifest is sorted, and stuff in files/ is
@@ -232,7 +361,7 @@ process_dir(const char *dir)
 					fprintf(stderr, "failed to write to %s/.Manifest.new: %s\n",
 							dir, strerror(errno));
 					fclose(f);
-					return;
+					return NULL;
 				}
 		}
 		fclose(f);
@@ -246,13 +375,13 @@ process_dir(const char *dir)
 					continue;
 				if (strcmp(e->d_name + strlen(e->d_name) - 7, ".ebuild") != 0)
 					continue;
-				write_hashes(dir, e->d_name, "EBUILD", m);
+				write_hashes(dir, e->d_name, "EBUILD", m, NULL);
 			}
 			closedir(d);
 		}
 
-		write_hashes(dir, "ChangeLog", "MISC", m);
-		write_hashes(dir, "metadata.xml", "MISC", m);
+		write_hashes(dir, "ChangeLog", "MISC", m, NULL);
+		write_hashes(dir, "metadata.xml", "MISC", m, NULL);
 
 		fflush(m);
 		fclose(m);
@@ -263,6 +392,8 @@ process_dir(const char *dir)
 			utimes(manifest, tv);
 			utimes(dir, tv);
 		}
+
+		return str_manifest;
 	}
 }
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f190ba6f3d24b7f0badd2ce9a61c039382107f56
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 27 14:10:46 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Nov 27 14:10:46 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f190ba6f

update-rsync-master: upgrade manifest signature hash from sha256 to sha512

 scripts/rsync-generation/update-rsync-master.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index a6c0cf7572..5f73206eae 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -212,7 +212,7 @@ sign_manifest() {
 		# remember, HOME is set to misc/ so .gnupg keychain lives there
 		gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
 			--pinentry-mode loopback \
-			--sign --clearsign --digest-algo SHA256 \
+			--sign --clearsign --digest-algo SHA512 \
 			--yes "${MANIFEST_CACHE}"/${mc} \
 			< "${BASE_PATH}"/autosigner.pwd >& /dev/null
 		if [[ -f ${MANIFEST_CACHE}/${mc}.asc ]] ; then

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     192190568f7ef7b98695c88b20d92ef1e90b516b
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 27 13:06:55 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Nov 27 13:06:55 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=19219056

hashgen: activate new hash set

 scripts/rsync-generation/hashgen.c | 65 +++++++++++++++++++++++++++-----------
 1 file changed, 47 insertions(+), 18 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index fed99a3132..ddc52752b4 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -20,6 +20,15 @@
  *   ${CC} -o hashgen -fopenmp ${CFLAGS} -lssl -lcrypto -lb2 hashgen.c
  */
 
+enum hash_impls {
+	HASH_SHA256    = 1<<0,
+	HASH_SHA512    = 1<<1,
+	HASH_WHIRLPOOL = 1<<2,
+	HASH_BLAKE2B   = 1<<3
+};
+/* default changed from sha256, sha512, whirlpool to blake2b, sha512 */
+static int hashes = HASH_BLAKE2B | HASH_SHA512;
+
 static inline void
 hex_hash(char *out, const unsigned char *buf, const int length)
 {
@@ -61,19 +70,23 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 		{
 #pragma omp section
 			{
-				SHA256_Update(&s256, data, len);
+				if (hashes & HASH_SHA256)
+					SHA256_Update(&s256, data, len);
 			}
 #pragma omp section
 			{
-				SHA512_Update(&s512, data, len);
+				if (hashes & HASH_SHA512)
+					SHA512_Update(&s512, data, len);
 			}
 #pragma omp section
 			{
-				WHIRLPOOL_Update(&whrl, data, len);
+				if (hashes & HASH_WHIRLPOOL)
+					WHIRLPOOL_Update(&whrl, data, len);
 			}
 #pragma omp section
 			{
-				blake2b_update(&bl2b, data, len);
+				if (hashes & HASH_BLAKE2B)
+					blake2b_update(&bl2b, data, len);
 			}
 		}
 	}
@@ -81,33 +94,49 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 #pragma omp parallel sections
 	{
 		{
-			unsigned char sha256buf[SHA256_DIGEST_LENGTH];
-			SHA256_Final(sha256buf, &s256);
-			hex_hash(sha256, sha256buf, SHA256_DIGEST_LENGTH);
+			if (hashes & HASH_SHA256) {
+				unsigned char sha256buf[SHA256_DIGEST_LENGTH];
+				SHA256_Final(sha256buf, &s256);
+				hex_hash(sha256, sha256buf, SHA256_DIGEST_LENGTH);
+			}
 		}
 #pragma omp section
 		{
-			unsigned char sha512buf[SHA512_DIGEST_LENGTH];
-			SHA512_Final(sha512buf, &s512);
-			hex_hash(sha512, sha512buf, SHA512_DIGEST_LENGTH);
+			if (hashes & HASH_SHA512) {
+				unsigned char sha512buf[SHA512_DIGEST_LENGTH];
+				SHA512_Final(sha512buf, &s512);
+				hex_hash(sha512, sha512buf, SHA512_DIGEST_LENGTH);
+			}
 		}
 #pragma omp section
 		{
-			unsigned char whrlplbuf[WHIRLPOOL_DIGEST_LENGTH];
-			WHIRLPOOL_Final(whrlplbuf, &whrl);
-			hex_hash(whrlpl, whrlplbuf, WHIRLPOOL_DIGEST_LENGTH);
+			if (hashes & HASH_WHIRLPOOL) {
+				unsigned char whrlplbuf[WHIRLPOOL_DIGEST_LENGTH];
+				WHIRLPOOL_Final(whrlplbuf, &whrl);
+				hex_hash(whrlpl, whrlplbuf, WHIRLPOOL_DIGEST_LENGTH);
+			}
 		}
 #pragma omp section
 		{
-			unsigned char blak2bbuf[BLAKE2B_OUTBYTES];
-			blake2b_final(&bl2b, blak2bbuf, BLAKE2B_OUTBYTES);
-			hex_hash(blak2b, blak2bbuf, WHIRLPOOL_DIGEST_LENGTH);
+			if (hashes & HASH_BLAKE2B) {
+				unsigned char blak2bbuf[BLAKE2B_OUTBYTES];
+				blake2b_final(&bl2b, blak2bbuf, BLAKE2B_OUTBYTES);
+				hex_hash(blak2b, blak2bbuf, WHIRLPOOL_DIGEST_LENGTH);
+			}
 		}
 	}
 	fclose(f);
 
-	fprintf(m, "%s %s %zd SHA256 %s SHA512 %s WHIRLPOOL %s BLAKE2B %s\n",
-			type, name, flen, sha256, sha512, whrlpl, blak2b);
+	fprintf(m, "%s %s %zd",type, name, flen);
+	if (hashes & HASH_BLAKE2B)
+		fprintf(m, " BLAKE2B %s", blak2b);
+	if (hashes & HASH_SHA256)
+		fprintf(m, " SHA256 %s", sha256);
+	if (hashes & HASH_SHA512)
+		fprintf(m, " SHA512 %s", sha512);
+	if (hashes & HASH_WHIRLPOOL)
+		fprintf(m, " WHIRLPOOL %s", whrlpl);
+	fprintf(m, "\n");
 }
 
 static char

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     2ced4dd4b98a186dbcef48501bdcbde57ecb40c1
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 27 12:35:20 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Nov 27 12:35:20 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=2ced4dd4

hashgen: phase 1, add BLAKE2B hashing

 scripts/rsync-generation/hashgen.c | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 6fc8e150d3..fed99a3132 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1,4 +1,4 @@
-/* Copyright 2006-2015 Gentoo Foundation; Distributed under the GPL v2 */
+/* Copyright 2006-2017 Gentoo Foundation; Distributed under the GPL v2 */
 #include <stdio.h>
 #include <string.h>
 #include <strings.h>
@@ -9,9 +9,16 @@
 #include <sys/time.h>
 #include <openssl/sha.h>
 #include <openssl/whrlpool.h>
+#include <blake2.h>
 
 /* Generate thick Manifests based on thin Manifests */
-/* gcc -o hashgen -fopenmp -Wall -Werror -O3 -pipe -lssl -lcrypto hashgen.c */
+
+/* In order to build this program, the following packages are required:
+ * - app-crypt/libb2 (for BLAKE2, for as long as openssl doesn't include it)
+ * - dev-libs/openssl (for SHA, WHIRLPOOL)
+ * compile like this
+ *   ${CC} -o hashgen -fopenmp ${CFLAGS} -lssl -lcrypto -lb2 hashgen.c
+ */
 
 static inline void
 hex_hash(char *out, const unsigned char *buf, const int length)
@@ -31,11 +38,13 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 	char sha256[(SHA256_DIGEST_LENGTH * 2) + 1];
 	char sha512[(SHA512_DIGEST_LENGTH * 2) + 1];
 	char whrlpl[(WHIRLPOOL_DIGEST_LENGTH * 2) + 1];
+	char blak2b[(BLAKE2B_OUTBYTES * 2) + 1];
 	char data[8096];
 	size_t len;
 	SHA256_CTX s256;
 	SHA512_CTX s512;
 	WHIRLPOOL_CTX whrl;
+	blake2b_state bl2b;
 
 	snprintf(fname, sizeof(fname), "%s/%s", root, name);
 	if ((f = fopen(fname, "r")) == NULL)
@@ -44,6 +53,7 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 	SHA256_Init(&s256);
 	SHA512_Init(&s512);
 	WHIRLPOOL_Init(&whrl);
+	blake2b_init(&bl2b, BLAKE2B_OUTBYTES);
 
 	while ((len = fread(data, 1, sizeof(data), f)) > 0) {
 		flen += len;
@@ -61,6 +71,10 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 			{
 				WHIRLPOOL_Update(&whrl, data, len);
 			}
+#pragma omp section
+			{
+				blake2b_update(&bl2b, data, len);
+			}
 		}
 	}
 
@@ -83,11 +97,17 @@ write_hashes(const char *root, const char *name, const char *type, FILE *m)
 			WHIRLPOOL_Final(whrlplbuf, &whrl);
 			hex_hash(whrlpl, whrlplbuf, WHIRLPOOL_DIGEST_LENGTH);
 		}
+#pragma omp section
+		{
+			unsigned char blak2bbuf[BLAKE2B_OUTBYTES];
+			blake2b_final(&bl2b, blak2bbuf, BLAKE2B_OUTBYTES);
+			hex_hash(blak2b, blak2bbuf, WHIRLPOOL_DIGEST_LENGTH);
+		}
 	}
 	fclose(f);
 
-	fprintf(m, "%s %s %zd SHA256 %s SHA512 %s WHIRLPOOL %s\n",
-			type, name, flen, sha256, sha512, whrlpl);
+	fprintf(m, "%s %s %zd SHA256 %s SHA512 %s WHIRLPOOL %s BLAKE2B %s\n",
+			type, name, flen, sha256, sha512, whrlpl, blak2b);
 }
 
 static char
@@ -149,7 +169,7 @@ process_dir(const char *dir)
 		struct timeval tv[2];
 
 		/* set mtime of Manifest to the one of the parent dir, this way
-		 * we enure the Manifest gets mtime bumped upon any change made
+		 * we ensure the Manifest gets mtime bumped upon any change made
 		 * to the directory, that is, a DIST change (Manifest itself) or
 		 * any other change (ebuild, files, metadata) */
 		if (stat(dir, &s)) {

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     76d1071b4ce6932900a7d668dfcf0d6563c710d4
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Sep  9 18:39:13 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Sep  9 18:39:13 2017 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=76d1071b

scripts/rsync-generation: git clean before git pull to cleanup untracked files

 scripts/rsync-generation/update-rsync-master.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 77ddd69f33..a6c0cf7572 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -80,6 +80,7 @@ GLOBALSTART=${START}
 echo "($(date +"%F %R")) updating DTDs"
 pushd "$DTDDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git clean -fq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -93,6 +94,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/dtd/timestamp.ch
 echo "($(date +"%F %R")) updating GLSAs"
 pushd "$GLSADIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git clean -fq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -106,6 +108,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/glsa/timestamp.c
 echo "($(date +"%F %R")) updating news"
 pushd "$NEWSDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git clean -fq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -133,6 +136,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating the gx86 tree"
 pushd "${GENTOOX86DIR}" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git clean -fq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"
@@ -158,6 +162,7 @@ START=$(date +%s)
 echo "($(date +"%F %R")) updating Prefix tree (Git image)"
 pushd "$PREFIXTREEDIR" || exit 1
 fromcommit=$(git log --pretty=format:'%H' -n1)
+git clean -fq
 git pull -q
 tocommit=$(git log --pretty=format:'%H' -n1)
 apply_git_mtimes "${fromcommit}" "${tocommit}"

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     79b055056b390f2eaf92b2a60a7a9dfe87c0c1c6
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 12 07:23:07 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Oct 12 07:23:07 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=79b05505

update-rsync-master: new gpg will ask for password even with --batch

 scripts/rsync-generation/update-rsync-master.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index c9721c3..46c031d 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -157,6 +157,7 @@ sign_manifest() {
 		cat "${RSYNCDIR}/${pkg}"/Manifest > "${MANIFEST_CACHE}"/${mc}
 		# remember, HOME is set to misc/ so .gnupg keychain lives there
 		gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C \
+			--pinentry-mode loopback \
 			--sign --clearsign --digest-algo SHA256 \
 			--yes "${MANIFEST_CACHE}"/${mc} \
 			< "${BASE_PATH}"/autosigner.pwd >& /dev/null

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     62444e8d9f2c4d2e33a4bb281b29b44f2491ae88
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Sep  9 13:36:37 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Sep  9 13:36:37 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=62444e8d

s/r-gen/update-rsync-master: don't add aliases to layout.conf

Portage triggers some weird path where it duplicates the repo conf under
a different name, leading to validation errors lateron.  It seems we
don't need this hack anymore, so remove it.

 scripts/rsync-generation/update-rsync-master.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 14ffc0b..c9721c3 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -191,12 +191,6 @@ TIME_MANISIGN=$((STOP - START))
 echo "($(date +"%F %R")) setting repo_name and making the prefix profiles development ones (iso exp)"
 echo "gentoo_prefix" > "${RSYNCDIR}"/profiles/repo_name
 touch -r "${CVSDIR}"/profiles/repo_name "${RSYNCDIR}"/profiles/repo_name
-{
-	echo ""
-	echo "# Let the Prefix tree satisfy depends on the main gx86 tree"
-	echo "aliases = gentoo"
-} >> "${RSYNCDIR}"/metadata/layout.conf
-touch -r "${CVSDIR}"/metadata/layout.conf "${RSYNCDIR}"/metadata/layout.conf
 # reset Prefix profiles to dev status
 sed -i -e '/prefix/s/exp/dev/' "${RSYNCDIR}"/profiles/profiles.desc
 touch -r "${CVSDIR}"/profiles/profiles.desc "${RSYNCDIR}"/profiles/profiles.desc

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     23171617759cdc31d16a18f38631ad177f4cc2a8
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Sep  7 11:01:52 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Sep  7 11:01:52 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=23171617

scripts/rsync-gen/refresh-mirror: echo we're about to kill procs

 scripts/rsync-generation/refresh-mirror.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index 348f2bf..8a12061 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -21,6 +21,7 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 	if [[ $(tail -n1 ${LOGFILE}) == *"rsync done" ]] ; then
 		pid=$(head -n1 ${LOGFILE})
 		if [[ ${pid} -gt 0 ]] ; then
+			echo "Killing stray/stuck process"
 			pstree -A -c -p ${pid} | grep -o '[0-9]\+' | xargs kill
 			rm /tmp/rsync-master-busy
 		fi

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     1ed42685935cd83f562c8c8f5c86522d35723189
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 17 04:26:03 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Aug 17 04:26:03 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=1ed42685

scripts/rsync-generation/refresh-mirror: store pid in a permanent place

 scripts/rsync-generation/refresh-mirror.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index d444d40..348f2bf 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -10,14 +10,16 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 	# allow one run to be skipped quietly
 	if [[ $((laststart + (40 * 60))) -lt ${now} ]] ; then
 		echo "another rsync-master generation process is still busy"
-		type pstree > /dev/null && pstree -p $(< /tmp/rsync-master-busy)
+		type pstree > /dev/null && pstree -p $(head -n1 ${LOGFILE})
 		ps -ef | grep '[r]efresh-mirror'
 		tail ${LOGFILE}
+	else
+		exit 0
 	fi
 	# if the log reports done, kill it as it seems that for some reason
 	# it hangs after doing this
 	if [[ $(tail -n1 ${LOGFILE}) == *"rsync done" ]] ; then
-		pid=$(< /tmp/rsync-master-busy)
+		pid=$(head -n1 ${LOGFILE})
 		if [[ ${pid} -gt 0 ]] ; then
 			pstree -A -c -p ${pid} | grep -o '[0-9]\+' | xargs kill
 			rm /tmp/rsync-master-busy
@@ -26,8 +28,9 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 else
 	mv ${LOGFILE} ${LOGFILE%.log}-prev.log
 	cd "$(readlink -f "${BASH_SOURCE[0]%/*}")"
-	echo $$ > /tmp/rsync-master-busy
-	echo "starting generation $(date)" > ${LOGFILE}
+	touch /tmp/rsync-master-busy
+	echo $$ > ${LOGFILE}
+	echo "starting generation $(date)" >> ${LOGFILE}
 	genandpush() {
 		./update-rsync-master.sh \
 			&& ./push-rsync1.sh

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     28d259b967dc8009033279b915bd25344bb59372
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 16 07:57:21 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Tue Aug 16 07:57:21 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=28d259b9

scripts/rsync-generation/refresh-mirror.sh: try to remediate hangs

 scripts/rsync-generation/refresh-mirror.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index 4020553..d444d40 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -10,13 +10,23 @@ if [[ -f /tmp/rsync-master-busy ]] ; then
 	# allow one run to be skipped quietly
 	if [[ $((laststart + (40 * 60))) -lt ${now} ]] ; then
 		echo "another rsync-master generation process is still busy"
+		type pstree > /dev/null && pstree -p $(< /tmp/rsync-master-busy)
 		ps -ef | grep '[r]efresh-mirror'
 		tail ${LOGFILE}
 	fi
+	# if the log reports done, kill it as it seems that for some reason
+	# it hangs after doing this
+	if [[ $(tail -n1 ${LOGFILE}) == *"rsync done" ]] ; then
+		pid=$(< /tmp/rsync-master-busy)
+		if [[ ${pid} -gt 0 ]] ; then
+			pstree -A -c -p ${pid} | grep -o '[0-9]\+' | xargs kill
+			rm /tmp/rsync-master-busy
+		fi
+	fi
 else
 	mv ${LOGFILE} ${LOGFILE%.log}-prev.log
 	cd "$(readlink -f "${BASH_SOURCE[0]%/*}")"
-	touch /tmp/rsync-master-busy
+	echo $$ > /tmp/rsync-master-busy
 	echo "starting generation $(date)" > ${LOGFILE}
 	genandpush() {
 		./update-rsync-master.sh \

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     e23dff0039387660848650a0a7ca0099456a7b4e
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 29 09:01:37 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jul 29 09:01:37 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=e23dff00

scripts/rsync-generation/refresh-mirror: fix syntax

 scripts/rsync-generation/refresh-mirror.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index bcfdd8f..4020553 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -27,7 +27,7 @@ else
 	}
 	# get a free filedescriptor in FD
 	exec {FD}>/tmp/rsync-master-busy
-	(((genandpush | tee -a "${LOGFILE}") ${FD}>&1 1>&2 2>&${FD} \
+	(((genandpush | tee -a "${LOGFILE}") {FD}>&1 1>&2 2>&${FD} \
 	    | tee -a "${LOGFILE}") 2> /dev/null)
 	echo "generation done $(date)" >> ${LOGFILE}
 	exec {FD}>&-

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     74d377662c78a5d6feef51e6bd9bbd38f9acc85c
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 29 08:08:06 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jul 29 08:08:06 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=74d37766

scripts/rsync-generation/refresh-mirror: use available filedescriptor iso fixed 3

 scripts/rsync-generation/refresh-mirror.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index 24643ca..bcfdd8f 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -25,8 +25,11 @@ else
 #			&& ./gen-timing-rsync0-graph.sh \
 #			&& popd > /dev/null
 	}
-	(((genandpush | tee -a "${LOGFILE}") 3>&1 1>&2 2>&3 \
+	# get a free filedescriptor in FD
+	exec {FD}>/tmp/rsync-master-busy
+	(((genandpush | tee -a "${LOGFILE}") ${FD}>&1 1>&2 2>&${FD} \
 	    | tee -a "${LOGFILE}") 2> /dev/null)
 	echo "generation done $(date)" >> ${LOGFILE}
+	exec {FD}>&-
 	rm -f /tmp/rsync-master-busy
 fi

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     e2f28a9e1fe07104ebed85f8c90e5ce5b928794f
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 18:35:43 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Tue May  3 18:35:43 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=e2f28a9e

update-rsync-master: send stats to graphite

 scripts/rsync-generation/update-rsync-master.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 1acdacd..14ffc0b 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -243,5 +243,13 @@ chmod -R u-s,g-s "${RSYNCDIR}"/metadata
 STOP=$(date +%s)
 TIME_TOTAL=$((STOP - GLOBALSTART))
 
-echo "${GLOBALSTART}	\"$(date +"%F %H:%M" -d @${GLOBALSTART})\"	${TIME_METADATA}	${TIME_SVNPREFIX}	${TIME_CVSGX86} ${TIME_EGENCACHE}	${TIME_TOTAL}	${TIME_MANISIGN}" >> \
-	/export/gentoo/statistics/stats/timing-rsync0.data
+# feed timings to graphite
+prefix="gentoo.rsync-generation.$(hostname -s)"
+{
+	echo "${prefix}.pull-metadata ${TIME_METADATA} ${GLOBALSTART}"
+	echo "${prefix}.pull-overlay ${TIME_SVNPREFIX} ${GLOBALSTART}"
+	echo "${prefix}.pull-gx86 ${TIME_CVSGX86} ${GLOBALSTART}"
+	echo "${prefix}.egencache ${TIME_EGENCACHE} ${GLOBALSTART}"
+	echo "${prefix}.wallclock ${TIME_TOTAL} ${GLOBALSTART}"
+	echo "${prefix}.signing ${TIME_MANISIGN} ${GLOBALSTART}"
+} | nc -q 0 localhost 3002

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     53aba1299c0c1c97839aebe1effda6672051e4c2
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 16:08:14 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Tue May  3 16:08:14 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=53aba129

hashgen: include sys/time.h for utimes

 scripts/rsync-generation/hashgen.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index 4dbd754..6fc8e15 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -6,6 +6,7 @@
 #include <errno.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <sys/time.h>
 #include <openssl/sha.h>
 #include <openssl/whrlpool.h>
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     93a59db592d00c3eb27f8f40da8e029144978af8
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 14 15:38:20 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Apr 14 15:38:20 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=93a59db5

scripts/rsync-generation: dtd and glsa will exist in repos of course

 scripts/rsync-generation/update-rsync-master.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index a58a997..1acdacd 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -7,8 +7,8 @@ BASE_PATH="$(readlink -f "${BASH_SOURCE[0]%/*}")"
 
 HGDIR="${BASE_PATH}/repos/prefix-tree"
 CVSDIR="${BASE_PATH}/repos/gentoo-x86"
-DTDDIR="${BASE_PATH}/dtd"
-GLSADIR="${BASE_PATH}/glsa"
+DTDDIR="${BASE_PATH}/repos/dtd"
+GLSADIR="${BASE_PATH}/repos/glsa"
 NEWSDIR="${BASE_PATH}/repos/gentoo-news"
 RSYNCDIR="${BASE_PATH}/master-rsync-tree"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     26342aaf3832adcbe6ec83da54a068d589e7220a
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 14 13:39:20 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Apr 14 13:39:20 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=26342aaf

scripts/rsync-generation: drop cvs usage for dtd and glsa, update to projects.xml

 scripts/rsync-generation/update-rsync-master.sh | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 753edb5..a58a997 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -7,10 +7,9 @@ BASE_PATH="$(readlink -f "${BASH_SOURCE[0]%/*}")"
 
 HGDIR="${BASE_PATH}/repos/prefix-tree"
 CVSDIR="${BASE_PATH}/repos/gentoo-x86"
-DTDDIR="${BASE_PATH}/repos/xml/htdocs/dtd"
-GLSADIR="${BASE_PATH}/repos/xml/htdocs/security/en/glsa"
+DTDDIR="${BASE_PATH}/dtd"
+GLSADIR="${BASE_PATH}/glsa"
 NEWSDIR="${BASE_PATH}/repos/gentoo-news"
-HERDSDIR="${BASE_PATH}/repos/xml/htdocs/proj/en/metastructure/herds"
 RSYNCDIR="${BASE_PATH}/master-rsync-tree"
 
 #### ---- Portage setup (use local modified copy) ---- ####
@@ -46,7 +45,7 @@ GLOBALSTART=${START}
 # update DTDs
 echo "($(date +"%F %R")) updating DTDs"
 pushd "$DTDDIR" || exit 1
-cvs -q update -dP
+git pull -q
 popd || exit 1
 # rsync the DTDs
 rsync -v --delete -aC "${DTDDIR}" "${RSYNCDIR}"/metadata/ || exit 1
@@ -56,7 +55,7 @@ echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/dtd/timestamp.ch
 # update GLSAs
 echo "($(date +"%F %R")) updating GLSAs"
 pushd "$GLSADIR" || exit 1
-cvs -q update -dP
+git pull -q
 popd || exit 1
 # rsync the GLSAs
 rsync -v --delete -aC "${GLSADIR}" "${RSYNCDIR}"/metadata/ || exit 1
@@ -73,13 +72,13 @@ rsync -v -Wa --exclude .git --delete "${NEWSDIR}" "${RSYNCDIR}"/metadata/news/
 date -R -u > "${RSYNCDIR}"/metadata/news/timestamp.chk
 echo "($(date +"%F %R")) set date to $(< "${RSYNCDIR}"/metadata/news/timestamp.chk)"
 
-# update herds
-echo "($(date +"%F %R")) updating herds.xml"
-pushd "${HERDSDIR}" || exit 1
-cvs -q update herds.xml
+# update projects
+echo "($(date +"%F %R")) updating projects.xml"
+pushd "${RSYNCDIR}"/metadata/ || exit 1
+rm -f projects.xml
+wget -q "https://api.gentoo.org/metastructure/projects.xml" || exit 1
 popd || exit 1
-rsync -v -a "${HERDSDIR}"/herds.xml "${RSYNCDIR}"/metadata/herds.xml || exit 1
-echo "($(date +"%F %R")) herds.xml updated"
+echo "($(date +"%F %R")) projectss.xml updated"
 
 STOP=$(date +%s)
 TIME_METADATA=$((STOP - START))

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     b0e71f5b8c5ca632fa44c4160fad2c69066d2c07
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  6 12:31:38 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Apr  6 12:31:38 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=b0e71f5b

update-rsync-master: use repositories-configuration with egencache

--portdir is deprecated

 scripts/rsync-generation/update-rsync-master.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index bfda169..753edb5 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -33,7 +33,7 @@ echo "(init) PATH=$PATH"
 
 #### ---- egencache settings ---- ####
 
-EGENCACHE_OPTS="--jobs=2 --load-average=3 --tolerant --update-use-local-desc"
+EGENCACHE_OPTS="--jobs=4 --load-average=3 --tolerant --update-use-local-desc"
 
 export PYTHONPATH PORTDIR PORTAGE_BASE_PATH PORTAGE_CONFIGROOT  \
 	ROOT PORTAGE_TMPFS FEATURES HOME
@@ -215,11 +215,19 @@ dolog() {
 dolog "${PORTAGE_BASE_PATH}/bin/egencache" --update --rsync \
 	--config-root="${PORTAGE_CONFIGROOT}" \
 	--cache-dir="${PORTAGE_DEPCACHEDIR}" \
-	--portdir="${RSYNCDIR}" \
 	--repo=gentoo_prefix \
+	--repositories-configuration='
+[DEFAULT]
+main-repo = gentoo_prefix
+
+[gentoo_prefix]
+location = '"${RSYNCDIR}"'
+sync-type = rsync
+sync-uri = rsync://dont-sync
+auto-sync = no
+' \
 	${EGENCACHE_OPTS} \
 	|| exit 5
-	#--repositories-configuration="/etc/repos.conf/gentoo_prefix.conf" \
 
 STOP=$(date +%s)
 TIME_EGENCACHE=$((STOP - START))

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     c91a896153fea22887e17de15b2fcb8b872dfcd5
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  6 11:25:00 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Apr  6 11:25:00 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=c91a8961

README: complete alternative install method

 scripts/rsync-generation/README.new-portage-usage | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/README.new-portage-usage b/scripts/rsync-generation/README.new-portage-usage
index 1fc4559..6f9511b 100644
--- a/scripts/rsync-generation/README.new-portage-usage
+++ b/scripts/rsync-generation/README.new-portage-usage
@@ -10,3 +10,4 @@ configure \
 	--with-portage-user="`id -un`" \
 	--with-portage-group="`id -gn`" \
 	--with-extra-path="/bin:/usr/bin"
+make install

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     07ccc6ebe2ffa8bc95b7ffbe26428bdfe090e392
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  6 11:28:05 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Apr  6 11:28:05 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=07ccc6eb

log egencache call

 scripts/rsync-generation/update-rsync-master.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 4e626b6..bfda169 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -208,7 +208,11 @@ START=$(date +%s)
 
 # generate the metadata
 echo "($(date +"%F %R")) generating metadata"
-"${PORTAGE_BASE_PATH}/bin/egencache" --update --rsync \
+dolog() {
+	echo $*
+	"$@"
+}
+dolog "${PORTAGE_BASE_PATH}/bin/egencache" --update --rsync \
 	--config-root="${PORTAGE_CONFIGROOT}" \
 	--cache-dir="${PORTAGE_DEPCACHEDIR}" \
 	--portdir="${RSYNCDIR}" \

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     ec694c9f2013e04049c8857a3627b803fabb33e6
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  6 10:49:38 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Apr  6 10:49:38 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=ec694c9f

keychain: force shell bash

 scripts/rsync-generation/update-rsync-master.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 126c919..4e626b6 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # get keys for ssh and signing
-eval $(keychain -q --noask --eval)
+eval $(env SHELL=/bin/bash keychain -q --noask --eval)
 
 BASE_PATH="$(readlink -f "${BASH_SOURCE[0]%/*}")"
 

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     90783e171220688a16eac338d2480d6625ab391b
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  6 10:49:12 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Apr  6 10:49:12 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=90783e17

hashgen: need string.h for strlen (Linux)

 scripts/rsync-generation/hashgen.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/rsync-generation/hashgen.c b/scripts/rsync-generation/hashgen.c
index e372eb0..4dbd754 100644
--- a/scripts/rsync-generation/hashgen.c
+++ b/scripts/rsync-generation/hashgen.c
@@ -1,5 +1,6 @@
 /* Copyright 2006-2015 Gentoo Foundation; Distributed under the GPL v2 */
 #include <stdio.h>
+#include <string.h>
 #include <strings.h>
 #include <dirent.h>
 #include <errno.h>

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     f01e7c990046fb703965e29e3e2b66e238164ee3
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Tue Jan  5 19:08:45 2016 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Tue Jan  5 19:08:45 2016 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=f01e7c99

update-rsync-master: remove some obsolete comments and code

 scripts/rsync-generation/update-rsync-master.sh | 40 -------------------------
 1 file changed, 40 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 9db1de5..126c919 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -27,10 +27,8 @@ HOME="${BASE_PATH}/misc"
 echo "(init) BASE_PATH=$BASE_PATH"
 echo "(init) PORTAGE_BASE_PATH=$PORTAGE_BASE_PATH"
 echo "(init) PYTHONPATH=$PYTHONPATH"
-#echo "(init) ROOT=$ROOT"
 echo "(init) PORTAGE_CONFIGROOT=$PORTAGE_CONFIGROOT"
 echo "(init) PORTAGE_DEPCACHEDIR=$PORTAGE_DEPCACHEDIR"
-#echo "(init) PORTAGE_PROFILE=$PORTAGE_PROFILE"
 echo "(init) PATH=$PATH"
 
 #### ---- egencache settings ---- ####
@@ -92,33 +90,6 @@ START=$(date +%s)
 
 echo "($(date +"%F %R")) updating the gx86 tree"
 pushd "${CVSDIR}" || exit 1
-# update the entire cvs tree in parallel
-#updatepids=
-#( cvs -q update -dPA [a-d]*-* 2>&1 | tee /var/tmp/cvsfubarlist.1 )&
-#( cvs -q update -dPA [a-m]*-* 2>&1 | tee /var/tmp/cvsfubarlist.1 )&
-#updatepids+=" $!"
-#( cvs -q update -dPA [e-r]*-* 2>&1 | tee /var/tmp/cvsfubarlist.2 )&
-#updatepids+=" $!"
-#( cvs -q update -dPA [s-z]*-* 2>&1 | tee /var/tmp/cvsfubarlist.3 )&
-#( cvs -q update -dPA [n-z]*-* 2>&1 | tee /var/tmp/cvsfubarlist.3 )&
-#updatepids+=" $!"
-# intentionally skip "scripts", since we overwrite with Prefix version
-#( cvs -q update -dPA eclass licenses metadata profiles virtual \
-#	header.txt skel.* 2>&1 | tee /var/tmp/cvsfubarlist.4 )&
-#updatepids+=" $!"
-#wait ${updatepids}
-
-# on principia: paralellism drains IO too much, so use single process
-#cvs -q update -dPA 2>&1 | tee /var/tmp/cvsfubarlist.all
-#echo "($(date +"%F %R")) cleansing stuff that looks wrong"
-#grep "^[M?C] " /var/tmp/cvsfubarlist.* | xargs --no-run-if-empty rm -v -Rf --
-#grep "^[M?C] " /var/tmp/cvsfubarlist.* | \
-#	sed -e 's/^..//g' -e 's/\/.*$//' | \
-#	xargs --no-run-if-empty cvs -q update -dPA 2>/dev/null || exit 1
-#rm -f /var/tmp/cvsfubarlist.*
-##find . -name ".#*" | xargs --no-run-if-empty rm --
-
-# with git above logic should be all unnecessary
 git pull -q
 popd || exit 1
 rsync -v \
@@ -180,12 +151,6 @@ sign_manifest() {
 	local mc=${pkg//\//_}.manifest
 	[[ -z ${pkg} ]] && return 1
 
-	if [[ $(<${RSYNCDIR}/${pkg}/Manifest) == *'BEGIN PGP SIGNED MESSAGE'* ]] ; then
-		rm -f ${MANIFEST_CACHE}/${mc}
-		# already signed, don't do anything
-		return 0
-	fi
-
 	if [[ ! -f ${MANIFEST_CACHE}/${mc} || ${RSYNCDIR}/${pkg}/Manifest -nt ${MANIFEST_CACHE}/${mc} ]] ; then
 		mkdir -p "${MANIFEST_CACHE}"
 
@@ -241,11 +206,6 @@ echo "($(date +"%F %R")) set up repo $(< "${RSYNCDIR}"/profiles/repo_name)"
 
 START=$(date +%s)
 
-# #410505 should not be necessary (even wrong)
-#echo "($(date +"%F %R")) cleaning metadata/cache"
-#rm -Rf "${RSYNCDIR}"/metadata/cache/*
-#rm -Rf "${RSYNCDIR}"/metadata/md5-cache/
-
 # generate the metadata
 echo "($(date +"%F %R")) generating metadata"
 "${PORTAGE_BASE_PATH}/bin/egencache" --update --rsync \

[gentoo-commits] repo/proj/prefix:master commit in: scripts/rsync-generation/

[Fabian Groffen]

commit:     4a14f8b1da0ff2087ca8dfe8913a4427a3c598e2
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 31 18:53:06 2015 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Mon Aug 31 18:53:06 2015 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=4a14f8b1

update-rsync-master: switch to prefix git tree

 scripts/rsync-generation/update-rsync-master.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/rsync-generation/update-rsync-master.sh b/scripts/rsync-generation/update-rsync-master.sh
index 485a2e7..9db1de5 100755
--- a/scripts/rsync-generation/update-rsync-master.sh
+++ b/scripts/rsync-generation/update-rsync-master.sh
@@ -139,9 +139,9 @@ TIME_CVSGX86=$((STOP - START))
 START=$(date +%s)
 
 # update the Mercurial image
-echo "($(date +"%F %R")) updating Prefix tree (Hg image)"
+echo "($(date +"%F %R")) updating Prefix tree (Git image)"
 pushd "$HGDIR" || exit 1
-hg pull -u || echo "Failed to pull!"
+git pull -q || echo "Failed to pull!"
 echo "($(date +"%F %R")) Mercurial image updated"
 
 # rsync the SVN image to the rsync master