[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210408-13-gcbefca9
Jon TURNEY
jturney@sourceware.org
Tue May 11 12:56:25 GMT 2021
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cbefca991dc088e2a53022a2aa2f4d92872f9db9
commit cbefca991dc088e2a53022a2aa2f4d92872f9db9
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 3 23:44:00 2021 +0100
Simplify package processing by removing vermap
Since a package now only contains either all source or all install
archives, we can drop the 'category' indirection that vermap provides.
Drop Package.vermap
Drop the category argument to Package.tar(), and drop checks that it's
the expected value before calling that.
Drop the filename index (which came from vermap) into Package.tars, and
rename it to Package.tarfiles ensure we've caught all references.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7fa131ab49af8b0d6188142a4b059f6b68777dcd
commit 7fa131ab49af8b0d6188142a4b059f6b68777dcd
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 3 23:22:05 2021 +0100
Use a temporary dict when validating curr: version mtime is latest
Use a temporary dict to hold mtime data when validating curr: version
mtime is latest, rather than bodging that into vermap.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1b5ec6d6b9349bb4da35063529d3587250748170
commit 1b5ec6d6b9349bb4da35063529d3587250748170
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 3 22:45:22 2021 +0100
Move some validation of tar file set for a package
Move some validation of tar file set for a package from
validate_packages() to read_one_package().
This is safe because merge() rejects merging packages when the same
version appears in the tar file list for both packages.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dcb1b09ae09b208d0f80b087c19c444eb8955505
commit dcb1b09ae09b208d0f80b087c19c444eb8955505
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 3 17:55:31 2021 +0100
Encapsulate access to package version list
Encapsulate access to the package version list, as a first step to
removing vermap.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d8869d2e727d4b10806fedca2c62f3105ce4f8b3
commit d8869d2e727d4b10806fedca2c62f3105ce4f8b3
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 3 17:47:46 2021 +0100
Drop validating versions appearing in override.hint keys
This is a followup to d863c1a2, 3cc08991 and 27852b24.
Since the curr:, prev:, test: keys are no longer permitted in
override.hint, there's no point in checking their value.
Do some simplifcation of the code which assigns versions to stability
levels, but there's probably more to do there.
Diff:
---
calm/package.py | 314 ++++++++++++++-------------------
calm/pkg2html.py | 25 +--
test/testdata/uploads/pkglist.expected | 12 +-
3 files changed, 151 insertions(+), 200 deletions(-)
diff --git a/calm/package.py b/calm/package.py
index b223edf..9c47f47 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -58,24 +58,26 @@ class Kind(Enum):
class Package(object):
def __init__(self):
self.pkgpath = '' # path to package, relative to arch
- self.tars = {}
+ self.tarfiles = {}
self.hints = {}
self.is_used_by = set()
self.version_hints = {}
self.override_hints = {}
self.skip = False
- self.vermap = {}
def __repr__(self):
return "Package('%s', %s, %s, %s, %s)" % (
self.pkgpath,
- pprint.pformat(self.tars),
+ pprint.pformat(self.tarfiles),
pprint.pformat(self.version_hints),
pprint.pformat(self.override_hints),
self.skip)
- def tar(self, vr, category):
- return self.tars[vr][self.vermap[vr][category]]
+ def tar(self, vr):
+ return self.tarfiles[vr]
+
+ def versions(self):
+ return self.tarfiles.keys()
# information we keep about a tar file
@@ -283,9 +285,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
(relpath, packages[p].pkgpath))
return True
- # determine version overrides
- note_absent = ('override.hint' in remove) or ('override.hint' in files)
-
if 'override.hint' in files:
# read override.hint
override_hints = read_hints(p, os.path.join(dirpath, 'override.hint'), hint.override)
@@ -296,13 +295,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
else:
override_hints = {}
- # if override.hint exists or is being removed, explicitly note absent
- # stability level hints
- if note_absent:
- for level in ['test', 'curr', 'prev']:
- if level not in override_hints:
- override_hints[level] = None
-
# read sha512.sum
sha512 = {}
if 'sha512.sum' not in files:
@@ -321,7 +313,7 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
# build a list of version-releases (since replacement pvr.hint files are
# allowed to be uploaded, we must consider both .tar and .hint files for
# that), and collect the attributes for each tar file
- tars = defaultdict(dict)
+ tars = {}
vr_list = set()
for f in list(files):
@@ -357,6 +349,23 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
vr_list.add(vr)
if not f.endswith('.hint'):
+ # a package can only contain tar archives of the appropriate type
+ if not re.search(r'-src.*\.tar', f):
+ if kind == Kind.source:
+ logging.error("source package '%s' has install archives" % (p))
+ return True
+ else:
+ if kind == Kind.binary:
+ logging.error("package '%s' has source archives" % (p))
+ return True
+
+ # for each version, a package can contain at most one tar file (of
+ # the appropriate type). Warn if we have too many (for e.g. both a
+ # .xz and .bz2 install tar file).
+ if vr in tars:
+ logging.error("package '%s' has more than one tar file for version '%s'" % (p, vr))
+ return True
+
# collect the attributes for each tar file
t = Tar()
t.path = relpath
@@ -371,7 +380,7 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
t.sha512 = sha512_file(os.path.join(dirpath, f))
logging.debug("no sha512.sum line for file %s in package '%s', computed sha512 hash is %s" % (f, p, t.sha512))
- tars[vr][f] = t
+ tars[vr] = t
# determine hints for each version we've encountered
version_hints = {}
@@ -410,12 +419,13 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
version_hints[ovr] = pvr_hint
hints[ovr] = hintobj
- actual_tars[ovr] = tars[vr]
+ if vr in tars:
+ actual_tars[ovr] = tars[vr]
packages[pn] = Package()
packages[pn].version_hints = version_hints
packages[pn].override_hints = override_hints
- packages[pn].tars = actual_tars
+ packages[pn].tarfiles = actual_tars
packages[pn].hints = hints
packages[pn].pkgpath = pkgpath
if kind == Kind.source:
@@ -573,46 +583,14 @@ def validate_packages(args, packages):
else:
logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p))
- packages[p].vermap = {}
- has_source = False
has_install = False
has_nonempty_install = False
- for vr in packages[p].tars:
- for (t, tar) in packages[p].tars[vr].items():
- # categorize each tarfile as either 'source' or 'install'
- if re.search(r'-src.*\.tar', t):
- category = 'source'
- has_source = True
- else:
- category = 'install'
- has_install = True
- if not packages[p].tars[vr][t].is_empty:
- has_nonempty_install = True
-
- if vr not in packages[p].vermap:
- packages[p].vermap[vr] = {}
-
- # for each version, a package can contain at most one source tar
- # file and at most one install tar file. warn if we have too many
- # (for e.g. both a .xz and .bz2 install tar file)
- if category in packages[p].vermap[vr]:
- logging.error("package '%s' has more than one %s tar file for version '%s'" % (p, category, vr))
- error = True
-
- # store tarfile corresponding to this version and category
- packages[p].vermap[vr][category] = t
- packages[p].vermap[vr]['mtime'] = tar.mtime
-
- # check that package only contains tar archives of the appropriate type
- if packages[p].kind == Kind.source:
- if has_install:
- logging.error("source package '%s' has install archives" % (p))
- error = True
- elif packages[p].kind == Kind.binary:
- if has_source:
- logging.error("package '%s' has source archives" % (p))
- error = True
+ if packages[p].kind == Kind.binary:
+ for vr in packages[p].versions():
+ has_install = True
+ if not packages[p].tar(vr).is_empty:
+ has_nonempty_install = True
obsolete = any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints])
@@ -637,53 +615,31 @@ def validate_packages(args, packages):
packages[p].skip = True
logging.info("package '%s' appears to be source-only as it has no non-empty install tarfiles and no dependencies, marking as 'skip'" % (p))
- # verify the versions specified for stability level exist
levels = ['test', 'curr', 'prev']
- for l in levels:
- if l in packages[p].override_hints:
- # check that if a version was specified, it exists
- v = packages[p].override_hints[l]
-
- if v is None:
- continue
-
- if v not in packages[p].vermap:
- logging.error("package '%s' stability '%s' selects non-existent version '%s'" % (p, l, v))
- error = True
# assign a version to each stability level
packages[p].stability = defaultdict()
# sort in order from highest to lowest version
- for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
+ for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True):
level_found = False
while True:
# no stability levels left
if len(levels) == 0:
- # XXX: versions which don't correspond to any stability level
- # should be reported, we might want to remove them at some point
- logging.log(5, "package '%s' has no stability levels left for version '%s'" % (p, v))
break
l = levels[0]
- # if current stability level has an override
- if (l in packages[p].override_hints) and (packages[p].override_hints[l] is not None):
- # if we haven't reached that version yet
- if v != packages[p].override_hints[l]:
- break
- else:
- logging.debug("package '%s' stability '%s' overridden to version '%s'" % (p, l, v))
# if package is explicitly marked as having that stability level
# (only used for test, currently)
- elif (l == 'test') and ('test' in packages[p].version_hints[v]):
+ if (l == 'test') and ('test' in packages[p].version_hints[v]):
logging.debug("package '%s' version '%s' marked as stability '%s'" % (p, v, l))
else:
- # level 'test' must be assigned by override
+ # level 'test' must be explicitly assigned
if l == 'test':
levels.remove(l)
- # go around again to check for override at the new level
+ # go around again to check at the new level
continue
else:
# if version is explicitly marked test, it can't be
@@ -714,7 +670,7 @@ def validate_packages(args, packages):
l = 'test'
if l not in packages[p].stability:
- for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
+ for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True):
if 'test' in packages[p].version_hints[v]:
packages[p].stability[l] = v
packages[p].version_hints[v][l] = ''
@@ -722,10 +678,10 @@ def validate_packages(args, packages):
# identify a 'best' version to take certain information from: this is
# the curr version, if we have one, otherwise, the highest version.
- if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap):
+ if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].versions()):
packages[p].best_version = packages[p].stability['curr']
- elif len(packages[p].vermap):
- packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0]
+ elif len(packages[p].versions()):
+ packages[p].best_version = sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True)[0]
else:
logging.error("package '%s' doesn't have any versions" % (p))
packages[p].best_version = None
@@ -742,17 +698,21 @@ def validate_packages(args, packages):
logging.warning("package '%s' doesn't have a curr version" % (p))
# error if the curr: version isn't the most recent non-test: version
- for v in sorted(packages[p].vermap.keys(), key=lambda v: packages[p].vermap[v]['mtime'], reverse=True):
+ mtimes = {}
+ for vr in packages[p].versions():
+ mtimes[vr] = packages[p].tar(vr).mtime
+
+ for v in sorted(packages[p].versions(), key=lambda v: mtimes[v], reverse=True):
if 'test' in packages[p].version_hints[v]:
continue
cv = packages[p].stability['curr']
- if cv not in packages[p].vermap:
+ if cv not in packages[p].versions():
continue
if cv != v:
- if packages[p].vermap[v]['mtime'] == packages[p].vermap[cv]['mtime']:
+ if mtimes[v] == mtimes[cv]:
# don't consider an equal mtime to be more recent
continue
@@ -784,47 +744,45 @@ def validate_packages(args, packages):
# If the install tarball is empty, we should probably either be marked
# obsolete (if we have no dependencies) or virtual (if we do)
if packages[p].kind == Kind.binary and not packages[p].skip:
- for vr in packages[p].vermap:
- if 'install' in packages[p].vermap[vr]:
- if packages[p].tar(vr, 'install').is_empty:
- # this classification relies on obsoleting packages
- # not being present in depends
- if packages[p].version_hints[vr].get('depends', ''):
- # also allow '_obsolete' because old obsoletion
- # packages depend on their replacement, but are not
- # obsoleted by it
- expected_categories = ['virtual', '_obsolete']
- else:
- expected_categories = ['_obsolete']
-
- if all(c not in packages[p].version_hints[vr].get('category', '').lower() for c in expected_categories):
- if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or
- ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
- lvl = logging.DEBUG
- else:
- lvl = logging.ERROR
- error = True
- logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in %s category" % (p, vr, expected_categories))
- # If the source tarball is empty, that can't be right!
- elif packages[p].kind == Kind.source:
- for vr in packages[p].vermap:
- if 'source' in packages[p].vermap[vr]:
- if packages[p].tar(vr, 'source').is_empty:
- if ((vr in past_mistakes.empty_source.get(p, [])) and
- '_obsolete' in packages[p].version_hints[vr].get('category', '')):
+ for vr in packages[p].versions():
+ if packages[p].tar(vr).is_empty:
+ # this classification relies on obsoleting packages
+ # not being present in depends
+ if packages[p].version_hints[vr].get('depends', ''):
+ # also allow '_obsolete' because old obsoletion
+ # packages depend on their replacement, but are not
+ # obsoleted by it
+ expected_categories = ['virtual', '_obsolete']
+ else:
+ expected_categories = ['_obsolete']
+
+ if all(c not in packages[p].version_hints[vr].get('category', '').lower() for c in expected_categories):
+ if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or
+ ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
lvl = logging.DEBUG
else:
- error = True
lvl = logging.ERROR
- logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr))
+ error = True
+ logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in %s category" % (p, vr, expected_categories))
+ # If the source tarball is empty, that can't be right!
+ elif packages[p].kind == Kind.source:
+ for vr in packages[p].versions():
+ if packages[p].tar(vr).is_empty:
+ if ((vr in past_mistakes.empty_source.get(p, [])) and
+ '_obsolete' in packages[p].version_hints[vr].get('category', '')):
+ lvl = logging.DEBUG
+ else:
+ error = True
+ lvl = logging.ERROR
+ logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr))
# make another pass to verify a source tarfile exists for every install
# tarfile version
for p in sorted(packages.keys()):
- for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
- if 'install' not in packages[p].vermap[v]:
- continue
+ if not packages[p].kind == Kind.binary:
+ continue
+ for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True):
sourceless = False
missing_source = True
@@ -837,14 +795,14 @@ def validate_packages(args, packages):
# mark the source tarfile as being used by an install tarfile
if es_p in packages:
- if v in packages[es_p].vermap and 'source' in packages[es_p].vermap[v]:
- packages[es_p].tar(v, 'source').is_used = True
+ if v in packages[es_p].versions():
+ packages[es_p].tar(v).is_used = True
packages[es_p].is_used_by.add(p)
missing_source = False
if missing_source:
# unless the install tarfile is empty
- if packages[p].tar(v, 'install').is_empty:
+ if packages[p].tar(v).is_empty:
sourceless = True
missing_source = False
@@ -854,7 +812,7 @@ def validate_packages(args, packages):
missing_source = False
# ... it's an error for this package to be missing source
- packages[p].tar(v, 'install').sourceless = sourceless
+ packages[p].tar(v).sourceless = sourceless
if missing_source:
logging.error("package '%s' version '%s' is missing source" % (p, v))
error = True
@@ -862,17 +820,17 @@ def validate_packages(args, packages):
# make another pass to verify that each non-empty source tarfile version has
# at least one corresponding non-empty install tarfile, in some package.
for p in sorted(packages.keys()):
- for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
- if 'source' not in packages[p].vermap[v]:
+ for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True):
+ if not packages[p].kind == Kind.source:
continue
- if packages[p].tar(v, 'source').is_empty:
+ if packages[p].tar(v).is_empty:
continue
if '_obsolete' in packages[p].version_hints[v].get('category', ''):
continue
- if not packages[p].tar(v, 'source').is_used:
+ if not packages[p].tar(v).is_used:
logging.error("package '%s' version '%s' source has no non-empty install tarfiles" % (p, v))
error = True
@@ -1073,10 +1031,10 @@ def write_setup_ini(args, packages, arch):
# (to maintain historical behaviour, include versions which only
# exist as a source package)
#
- versions = set(po.vermap.keys())
+ versions = set(po.versions())
sibling_src = pn + '-src'
if sibling_src in packages:
- versions.update(packages[sibling_src].vermap.keys())
+ versions.update(packages[sibling_src].versions())
for version in sorted(versions, key=lambda v: SetupVersion(v), reverse=True):
# skip over versions assigned to stability level: 'curr' has
@@ -1117,9 +1075,9 @@ def write_setup_ini(args, packages, arch):
print("version: %s" % version, file=f)
is_empty = False
- if 'install' in po.vermap.get(version, {}):
+ if version in po.versions():
tar_line(po, 'install', version, f)
- is_empty = po.tar(version, 'install').is_empty
+ is_empty = po.tar(version).is_empty
hints = po.version_hints.get(version, {})
@@ -1133,7 +1091,7 @@ def write_setup_ini(args, packages, arch):
# external-source points to a source file in another package
if s:
- if 'source' in packages[s].vermap.get(version, {}):
+ if version in packages[s].versions():
tar_line(packages[s], 'source', version, f)
else:
if not (is_empty or packages[s].orig_name in past_mistakes.self_source):
@@ -1173,7 +1131,7 @@ def write_setup_ini(args, packages, arch):
# helper function to output details for a particular tar file
def tar_line(p, category, v, f):
- to = p.tar(v, category)
+ to = p.tar(v)
fn = os.path.join(to.path, to.fn)
sha512 = to.sha512
size = to.size
@@ -1266,21 +1224,22 @@ def merge(a, *l):
c[p] = b[p]
# else, if the package is both in a and b, we have to do a merge
else:
+ # package must be of the same kind
+ if c[p].kind != b[p].kind:
+ logging.error("package '%s' is of more than one kind" % (p))
+ return None
+
# package must exist at same relative path
if c[p].pkgpath != b[p].pkgpath:
logging.error("package '%s' is at paths %s and %s" % (p, c[p].pkgpath, b[p].pkgpath))
return None
else:
- for vr in b[p].tars:
- if vr in c[p].tars:
- for t in b[p].tars[vr]:
- if t in c[p].tars[vr]:
- logging.error("package '%s' has duplicate tarfile %s for version %s" % (p, t, vr))
- return None
- else:
- c[p].tars[vr][t] = b[p].tars[vr][t]
+ for vr in b[p].tarfiles:
+ if vr in c[p].tarfiles:
+ logging.error("package '%s' has duplicate tarfile for version %s" % (p, vr))
+ return None
else:
- c[p].tars[vr] = b[p].tars[vr]
+ c[p].tarfiles[vr] = b[p].tarfiles[vr]
# hints from b override hints from a, but warn if they have
# changed
@@ -1316,15 +1275,10 @@ def delete(packages, path, fn):
for p in packages:
if packages[p].pkgpath == pkgpath:
- for vr in packages[p].tars:
- for t in packages[p].tars[vr]:
- if t == fn:
- del packages[p].tars[vr][t]
- break
-
- # if no packages remain for this vr, also remove from vermap
- if not packages[p].tars[vr]:
- packages[p].vermap.pop(vr, None)
+ for vr in packages[p].tarfiles:
+ if packages[p].tarfiles[vr].fn == fn:
+ del packages[p].tarfiles[vr]
+ break
for h in packages[p].hints:
if packages[p].hints[h].fn == fn:
@@ -1333,7 +1287,7 @@ def delete(packages, path, fn):
break
# if nothing remains, also remove from package set
- if not packages[p].vermap and not packages[p].hints:
+ if not packages[p].tarfiles and not packages[p].hints:
ex_packages.append(p)
# (modify package set outside of iteration over it)
@@ -1367,8 +1321,8 @@ def is_in_package_list(ppath, plist):
#
def mark_package_fresh(packages, p, v):
- if 'install' in packages[p].vermap[v]:
- packages[p].tar(v, 'install').fresh = True
+ if packages[p].kind == Kind.binary:
+ packages[p].tar(v).fresh = True
# ... mark any corresponding sibling or external-source package version as also fresh
if 'external-source' in packages[p].version_hints[v]:
@@ -1377,9 +1331,9 @@ def mark_package_fresh(packages, p, v):
es_p = p + '-src'
if es_p in packages:
- if v in packages[es_p].vermap:
- if 'source' in packages[es_p].vermap[v]:
- packages[es_p].tar(v, 'source').fresh = True
+ if v in packages[es_p].versions():
+ if packages[es_p].kind == Kind.source:
+ packages[es_p].tar(v).fresh = True
#
@@ -1396,7 +1350,7 @@ def stale_packages(packages):
# mark any versions explicitly listed in the keep: override hint
for v in po.override_hints.get('keep', '').split():
- if v in po.vermap:
+ if v in po.versions():
mark_package_fresh(packages, pn, v)
else:
logging.error("package '%s' has non-existent keep: version '%s'" % (pn, v))
@@ -1404,7 +1358,7 @@ def stale_packages(packages):
# mark as fresh the highest n non-test versions, where n is given by the
# keep-count: override hint, (defaulting to DEFAULT_KEEP_COUNT)
keep_count = int(po.override_hints.get('keep-count', common_constants.DEFAULT_KEEP_COUNT))
- for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
+ for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True):
if 'test' not in po.version_hints[v]:
if keep_count <= 0:
break
@@ -1414,7 +1368,7 @@ def stale_packages(packages):
# mark as fresh the highest n test versions, where n is given by the
# keep-count-test: override hint, (defaulting to DEFAULT_KEEP_COUNT_TEST)
keep_count = int(po.override_hints.get('keep-count-test', common_constants.DEFAULT_KEEP_COUNT_TEST))
- for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v), reverse=True):
+ for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True):
if 'test' in po.version_hints[v]:
if keep_count <= 0:
break
@@ -1427,11 +1381,10 @@ def stale_packages(packages):
# it is included)
keep_days = po.override_hints.get('keep-days', common_constants.DEFAULT_KEEP_DAYS)
newer = False
- for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v)):
+ for v in sorted(po.versions(), key=lambda v: SetupVersion(v)):
if not newer:
- if 'install' in po.vermap[v]:
- if po.tar(v, 'install').mtime > (time.time() - (keep_days * 24 * 60 * 60)):
- newer = True
+ if po.tar(v).mtime > (time.time() - (keep_days * 24 * 60 * 60)):
+ newer = True
if newer:
mark_package_fresh(packages, pn, v)
@@ -1441,16 +1394,14 @@ def stale_packages(packages):
for pn, po in packages.items():
all_stale = {}
- for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v)):
+ for v in sorted(po.versions(), key=lambda v: SetupVersion(v)):
all_stale[v] = True
- for category in ['source', 'install']:
- if category in po.vermap[v]:
- if not getattr(po.tar(v, category), 'fresh', False):
- to = po.tar(v, category)
- stale.add(to.path, to.fn)
- logging.debug("package '%s' version '%s' %s is stale" % (pn, v, category))
- else:
- all_stale[v] = False
+ if not getattr(po.tar(v), 'fresh', False):
+ to = po.tar(v)
+ stale.add(to.path, to.fn)
+ logging.debug("package '%s' version '%s' is stale" % (pn, v))
+ else:
+ all_stale[v] = False
for v in po.hints:
# if there's a pvr.hint without a fresh source or install of the
@@ -1461,12 +1412,11 @@ def stale_packages(packages):
logging.debug("package '%s' version '%s' hint is stale" % (pn, v))
# clean up freshness mark
- for v in po.vermap:
- for c in ['source', 'install']:
- try:
- delattr(po.tar(v, c), 'fresh')
- except (KeyError, AttributeError):
- pass
+ for v in po.versions():
+ try:
+ delattr(po.tar(v), 'fresh')
+ except AttributeError:
+ pass
return stale
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index ffd16b0..80bc393 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -42,7 +42,6 @@
import argparse
import glob
import html
-import itertools
import logging
import math
import os
@@ -266,19 +265,20 @@ def update_package_listings(args, packages):
print('<table class="pkgtable">', file=f)
print('<tr><th>Version</th><th>Package Size</th><th>Date</th><th>Files</th><th>Status</th></tr>', file=f)
- def tar_line(pn, p, category, v, arch, f):
- if category not in p.vermap[v]:
- return
- size = int(math.ceil(p.tar(v, category).size / 1024))
- name = v if category == 'install' else v + ' (source)'
- target = "%s-%s" % (p.orig_name, v) + ('' if category == 'install' else '-src')
+ def tar_line(pn, p, v, arch, f):
+ size = int(math.ceil(p.tar(v).size / 1024))
+ if p.kind == package.Kind.binary:
+ name = v
+ target = "%s-%s" % (p.orig_name, v)
+ else:
+ name = v + ' (source)'
+ target = "%s-%s-src" % (p.orig_name, v)
test = 'test' if 'test' in p.version_hints[v] else 'stable'
- ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v, category).mtime))
+ ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v).mtime))
print('<tr><td>%s</td><td class="right">%d KiB</td><td>%s</td><td>[<a href="../%s/%s/%s">list of files</a>]</td><td>%s</td></tr>' % (name, size, ts, arch, pn, target, test), file=f)
- for version in sorted(packages[arch][p].vermap.keys(), key=lambda v: SetupVersion(v)):
- tar_line(p, packages[arch][p], 'install', version, arch, f)
- tar_line(p, packages[arch][p], 'source', version, arch, f)
+ for version in sorted(packages[arch][p].versions(), key=lambda v: SetupVersion(v)):
+ tar_line(p, packages[arch][p], version, arch, f)
print('</table><br>', file=f)
print('</ul>', file=f)
@@ -444,7 +444,8 @@ def write_arch_listing(args, packages, arch):
else:
listings = []
- for tn, to in itertools.chain.from_iterable([packages[p].tars[vr].items() for vr in packages[p].tars]):
+ for to in packages[p].tarfiles.values():
+ tn = to.fn
fver = re.sub(r'\.tar.*$', '', tn)
listing = os.path.join(dirpath, fver)
diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected
index e5cbbbf..3b61113 100644
--- a/test/testdata/uploads/pkglist.expected
+++ b/test/testdata/uploads/pkglist.expected
@@ -1,4 +1,4 @@
-{'testpackage': Package('testpackage', {'1.0-1': {'testpackage-1.0-1.tar.bz2': Tar('testpackage-1.0-1.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test package"',
+{'testpackage': Package('testpackage', {'1.0-1': Tar('testpackage-1.0-1.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test package"',
'ldesc': '"A test package\n'
"It's description might contains some unicode junk\n"
'Like itâs youâre Markup Language⢠Nokogiriâs toolâthat '
@@ -6,7 +6,7 @@
'category': 'Devel',
'requires': 'cygwin',
'depends': 'cygwin'}}, {}, False),
- 'testpackage-src': Package('testpackage', {'1.0-1': {'testpackage-1.0-1-src.tar.bz2': Tar('testpackage-1.0-1-src.tar.bz2', 'x86/release/testpackage', 'acfd77df3347e6432ccf29c12989964bc680a158d574f85dfa7ef222759f411006c7bd2773e37c5abdee628bea769b2da9aae213db615cd91402fd385373933d', 266, False)}}, {'1.0-1': {'sdesc': '"A test package"',
+ 'testpackage-src': Package('testpackage', {'1.0-1': Tar('testpackage-1.0-1-src.tar.bz2', 'x86/release/testpackage', 'acfd77df3347e6432ccf29c12989964bc680a158d574f85dfa7ef222759f411006c7bd2773e37c5abdee628bea769b2da9aae213db615cd91402fd385373933d', 266, False)}, {'1.0-1': {'sdesc': '"A test package"',
'ldesc': '"A test package\n'
"It's description might contains some unicode junk\n"
'Like itâs youâre Markup Language⢠Nokogiriâs toolâthat '
@@ -15,20 +15,20 @@
'requires': 'cygwin',
'homepage': 'http://homepage.url',
'depends': 'cygwin'}}, {}, True),
- 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': {'testpackage-subpackage-1.0-1.tar.bz2': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test subpackage"',
+ 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"',
'ldesc': '"A test subpackage"',
'category': 'Devel',
'external-source': 'testpackage-src'}}, {}, False),
- 'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1.tar.zst': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}}, {'1.0-1': {'category': 'Base',
+ 'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}, {'1.0-1': {'category': 'Base',
'requires': '',
'sdesc': '"test package (zstd compressed)"',
'ldesc': '"test package (zstd compressed)"',
'depends': ''}}, {}, False),
- 'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1-src.tar.zst': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}}, {'1.0-1': {'category': 'Base',
+ 'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}, {'1.0-1': {'category': 'Base',
'build-depends': 'cygport',
'sdesc': '"test package (zstd compressed)"',
'ldesc': '"test package (zstd compressed)"',
'skip': ''}}, {}, True),
- 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}}, {'1.0-1': {'sdesc': '"A test subpackage 2"',
+ 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"',
'ldesc': '"A test subpackage 2"',
'category': 'Devel'}}, {}, False)}
More information about the Cygwin-apps-cvs
mailing list