From 47d2bc78adb49f3182f9a3d7a4baea363e772d64 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 6 Dec 2013 12:17:48 +0100 Subject: implement POC client-side merging of pdiffs via apt-file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea of pdiffs is to avoid downloading the hole file by patching the existing index. This works very well, but becomes slow if a lot of patches needs to be applied to reconstruct an up-to-date index and in recent years more and more dinstall (or similar) runs are executed creating more and more pdiffs in the same amount of time, so pdiffs became less useful. The solution is simple: Reduce the amount of patches (which are very small) which need to be applied on top of the index we have available (which is usually pretty big). This can be done in two ways: Either merge the patches on the server-side so that the client has to download only one patch or the patches are all downloaded and merged on the client-side. The first needs a client who is doing one step at a time who can also skip patches if it needs (APT supports this for a long time now). The later is implemented by this commit, but depends on the server NOT merging the patches and the patches being in a strict order in which no patch is skipped. This is traditionally the case for dak, but other repository creators support merging – e.g. reprepro (which helpfully adds a flag indicating that the patches are merged). To support both or even mixes a client needs more information which isn't available for now. This POC uses the external diffindex-rred included in apt-file to do the heavy lifting of merging & applying all patches in one pass, hence to test this feature apt-file needs to be installed. --- test/integration/test-pdiff-usage | 147 ++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 20 deletions(-) (limited to 'test') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index ac0563b7f..5a06e0ccb 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -5,39 +5,146 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' buildaptarchive setupflataptarchive changetowebserver -signreleasefiles -testsuccess aptget update -testnopackage newstuff PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" -testequal "$(cat ${PKGFILE}) + +echo '#!/bin/sh +touch merge-was-used +/usr/bin/diffindex-rred "$@"' > extrred +chmod +x extrred +echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred + +wasmergeused() { + testsuccess aptget update "$@" + msgtest 'Check if the right pdiff merger was used' + if [ -e ./merge-was-used ]; then + rm -f ./merge-was-used + if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then + msgpass + else + msgfail "Merge shouldn't have been used, but was" + fi + elif echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then + msgfail "Merge should have been used, but wasn't" + else + msgpass + fi +} + +testrun() { + # setup the base + find aptarchive -name 'Packages*' -type f -delete + cp ${PKGFILE} aptarchive/Packages + compressfile 'aptarchive/Packages' + generatereleasefiles + signreleasefiles + rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists + testsuccess aptget update "$@" + cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak + testnopackage newstuff + testequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff -cp ${PKGFILE}-new aptarchive/Packages -compressfile 'aptarchive/Packages' -rm -rf aptarchive/Packages.diff -mkdir -p aptarchive/Packages.diff -PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" -diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true -cat $PATCHFILE | gzip > ${PATCHFILE}.gz -PATCHINDEX="aptarchive/Packages.diff/Index" -echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) + # apply with one patch + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) SHA1-History: 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) SHA1-Patches: 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX -generatereleasefiles '+1hour' -signreleasefiles -find aptarchive -name 'Packages*' -type f -delete -testsuccess aptget update + generatereleasefiles '+1hour' + signreleasefiles + find aptarchive -name 'Packages*' -type f -delete + wasmergeused "$@" + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff -testnopackage oldstuff -testequal "$(cat ${PKGFILE}-new) + # index is already up-to-date + find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete + testsuccess aptget update "$@" + testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff + + # apply with two patches + cp ${PKGFILE}-new aptarchive/Packages + echo ' +Package: futurestuff +Version: 1.0 +Architecture: i386 +Maintainer: Joe Sixpack +Installed-Size: 202 +Filename: pool/futurestuff_1.0_i386.deb +Size: 202200 +MD5sum: 311aeeaaae5ba33aff1ceaf3e1f76671 +SHA1: 3c695e028f7a1ae324deeaae5ba332desa81088c +SHA256: b46fd154615edaae5ba33c56a5cc0e7deaef23e2da3e4f129727fd660f28f050 +Description: some cool and shiny future stuff + This package will appear in the next next mirror update +Description-md5: d5f89fbbc2ce34c455dfee9b67d82b6b' >> aptarchive/Packages + + compressfile 'aptarchive/Packages' + PATCHFILE2="aptarchive/Packages.diff/$(date -d 'now + 1hour' '+%Y-%m-%d-%H%M.%S')" + diff -e ${PKGFILE}-new aptarchive/Packages > ${PATCHFILE2} || true + cat $PATCHFILE2 | gzip > ${PATCHFILE2}.gz + echo "SHA1-Current: $(sha1sum aptarchive/Packages | cut -d' ' -f 1) $(stat -c%s aptarchive/Packages) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum ${PKGFILE} | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}) $(basename ${PATCHFILE}) + $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) $(basename ${PATCHFILE2}) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) + $(sha1sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX + generatereleasefiles '+2hour' + signreleasefiles + cp -a aptarchive/Packages Packages-future + find aptarchive -name 'Packages*' -type f -delete + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + wasmergeused "$@" + testnopackage oldstuff + testequal "$(cat Packages-future) +" aptcache show apt newstuff futurestuff + + # patch applying fails, but successful fallback + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + echo 'I am Mallory and I change files' >> $PATCHFILE + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + generatereleasefiles '+1hour' + signreleasefiles + testsuccess aptget update "$@" + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff +} + +testrun -o Debug::pkgAcquire::Diffs=1 -o Debug::pkgAcquire::rred=1 -o Acquire::PDiffs::Merge=0 +testrun -o Debug::pkgAcquire::Diffs=1 -o Debug::pkgAcquire::rred=1 -o Acquire::PDiffs::Merge=1 -- cgit v1.2.3-70-g09d2 From 9aaa45283b14c3c81641f3f3e38157a267b1e8f7 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 16:14:23 +0100 Subject: actually register the tempfile removal atexit Git-Dch: Ignore --- test/libapt/sourcelist_test.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'test') diff --git a/test/libapt/sourcelist_test.cc b/test/libapt/sourcelist_test.cc index 6e83d08e0..adadae6a7 100644 --- a/test/libapt/sourcelist_test.cc +++ b/test/libapt/sourcelist_test.cc @@ -36,6 +36,7 @@ int main(int argc, char *argv[]) ; FileFd fd; + atexit(remove_tmpfile); tempfile = strdup("apt-test.XXXXXXXX"); tempfile_fd = mkstemp(tempfile); -- cgit v1.2.3-70-g09d2 From 50bd6fd3794dd1f61185302129dc6cd218d20b98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 17:23:05 +0100 Subject: integrate Anthonys rred with POC for client-side merge Providing the benefits of both without the downsides :) (ABI breaks or external dependencies) For this Anthonys rred is equipped with: - magic-filename-pickup of patches rather than explicit messages - use of FileFd instead of FILE* to get on-the-fly uncompress of the gzip compressed pdiff patches The acquire code in turn stops checking for apt-file's helper as our own rred is now clever enough for our needs. --- apt-pkg/acquire-item.cc | 11 ++--- methods/rred.cc | 94 +++++++++++---------------------------- test/integration/test-pdiff-usage | 23 +++++++--- 3 files changed, 45 insertions(+), 83 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7f6443555..1185908f3 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -513,14 +513,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); if (pdiff_merge == true) { - // this perl script is provided by apt-file - pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred")); - if (pdiff_merge == true) - { - // reprepro adds this flag if it has merged patches on the server - std::string const precedence = Tags.FindS("X-Patch-Precedence"); - pdiff_merge = (precedence != "merged"); - } + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); } if (pdiff_merge == false) diff --git a/methods/rred.cc b/methods/rred.cc index ed3fcc82e..313166160 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -407,13 +407,13 @@ class Patch { public: - void read_diff(FILE *f) + void read_diff(FileFd &f) { char buffer[BLOCK_SIZE]; bool cmdwanted = true; Change ch(0); - while(fgets(buffer, sizeof(buffer), f)) + while(f.ReadLine(buffer, sizeof(buffer))) { if (cmdwanted) { char *m, *c; @@ -534,66 +534,11 @@ class Patch { } }; -bool LookupPatches(const std::string &Message, std::vector &lines) -{ - const char *Tag = "Patches"; - const size_t Length = strlen(Tag); - - std::string::const_iterator I, J; - - std::clog << "Looking for \"Patches:\" section in message:\n\n" << Message << "\n\n"; - std::clog.flush(); - - for (I = Message.begin(); I + Length < Message.end(); ++I) - { - if (I[Length] == ':' && stringcasecmp(I, I+Length, Tag) == 0) - { - // found the tag, now read the patches - for(;;) { - for (; I < Message.end() && *I != '\n'; ++I); - if (I < Message.end()) I++; - if (I == Message.end() || *I != ' ') - break; - while (I < Message.end() && isspace(*I)) I++; - for (J = I; J < Message.end() && *J != '\n'; ++J) - ; - do - J--; - while (I < J && isspace(*J)); - if (I < J) - lines.push_back(std::string(I,++J)); - else - break; - I = J; - } - std::clog << "Found " << lines.size() << " patches!\n"; - std::clog.flush(); - return true; - } - } - std::clog << "Found no patches! :(\n"; - std::clog.flush(); - return false; -} - - class RredMethod : public pkgAcqMethod { private: bool Debug; - std::vector patchpaths; protected: - virtual bool HandleMessage(int Number, std::string Message) { - if (Number == 600) - { - patchpaths.clear(); - LookupPatches(Message, patchpaths); - std::clog << "Ended up with " << patchpaths.size() << " patches!\n"; - std::clog.flush(); - } - return pkgAcqMethod::HandleMessage(Number, Message); - } - virtual bool Fetch(FetchItem *Itm) { Debug = _config->FindB("Debug::pkgAcquire::RRed", false); URI Get = Itm->Uri; @@ -601,17 +546,29 @@ class RredMethod : public pkgAcqMethod { FetchResult Res; Res.Filename = Itm->DestFile; - if (Itm->Uri.empty()) { + if (Itm->Uri.empty()) + { Path = Itm->DestFile; Itm->DestFile.append(".result"); } else URIStart(Res); + std::vector patchpaths; Patch patch; - if (patchpaths.empty()) - { + if (FileExists(Path + ".ed") == true) patchpaths.push_back(Path + ".ed"); + else + { + _error->PushToStack(); + std::vector patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false); + _error->RevertToStack(); + + std::string const baseName = Path + ".ed."; + for (std::vector::const_iterator p = patches.begin(); + p != patches.end(); ++p) + if (p->compare(0, baseName.length(), baseName) == 0) + patchpaths.push_back(*p); } std::string patch_name; @@ -624,13 +581,15 @@ class RredMethod : public pkgAcqMethod { std::clog << "Patching " << Path << " with " << patch_name << std::endl; - FILE *p = fopen(patch_name.c_str(), "r"); - if (p == NULL) { - std::clog << "Could not open patch file " << patch_name << std::endl; + FileFd p; + // all patches are compressed, even if the name doesn't reflect it + if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) { + std::cerr << "Could not open patch file " << patch_name << std::endl; + _error->DumpErrors(std::cerr); abort(); } patch.read_diff(p); - fclose(p); + p.Close(); } if (Debug == true) @@ -697,10 +656,9 @@ int main(int argc, char **argv) } for (; i < argc; i++) { - FILE *p; - p = fopen(argv[i], "r"); - if (!p) { - perror(argv[i]); + FileFd p; + if (p.Open(argv[i], FileFd::ReadOnly) == false) { + _error->DumpErrors(std::cerr); exit(1); } patch.read_diff(p); diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5a06e0ccb..ad31511b9 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -20,10 +20,19 @@ chmod +x extrred echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred wasmergeused() { - testsuccess aptget update "$@" + msgtest 'Test for successful execution of' "$*" + local OUTPUT=$(mktemp) + addtrap "rm $OUTPUT;" + if aptget update "$@" >${OUTPUT} 2>&1; then + msgpass + else + echo + cat $OUTPUT + msgfail + fi + msgtest 'Check if the right pdiff merger was used' - if [ -e ./merge-was-used ]; then - rm -f ./merge-was-used + if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then msgpass else @@ -50,7 +59,7 @@ testrun() { testequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff - # apply with one patch + msgmsg 'Testcase: apply with one patch' cp ${PKGFILE}-new aptarchive/Packages compressfile 'aptarchive/Packages' mkdir -p aptarchive/Packages.diff @@ -73,13 +82,13 @@ SHA1-Patches: testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # index is already up-to-date + msgmsg 'Testcase: index is already up-to-date' find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete testsuccess aptget update "$@" testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # apply with two patches + msgmsg 'Testcase: apply with two patches' cp ${PKGFILE}-new aptarchive/Packages echo ' Package: futurestuff @@ -120,7 +129,7 @@ SHA1-Patches: testequal "$(cat Packages-future) " aptcache show apt newstuff futurestuff - # patch applying fails, but successful fallback + msgmsg 'Testcase: patch applying fails, but successful fallback' rm -rf rootdir/var/lib/apt/lists cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists cp ${PKGFILE}-new aptarchive/Packages -- cgit v1.2.3-70-g09d2 From f74a6fa120759d0a1bd4a5aff0dc2c50911b5407 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 18:44:47 +0100 Subject: rework some testcases to not spit out text Rework also uncovers two FIXMEs Git-Dch: Ignore --- test/integration/framework | 22 ++++---- test/integration/test-apt-progress-fd-conffile | 17 +++++-- test/integration/test-apt-sources-deb822 | 70 ++++++++++++++------------ test/integration/test-bug-728500-tempdir | 19 +++++-- 4 files changed, 77 insertions(+), 51 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index 6ada1e9cc..2fe059280 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -948,22 +948,22 @@ testempty() { test -z "$($* 2>&1)" && msgpass || msgfail } -testequalwithmsg() { - local MSG="$1" - shift +testequal() { + local MSG='Test of equality of' + if [ "$1" = '--nomsg' ]; then + MSG='' + shift + fi + local COMPAREFILE=$(mktemp) addtrap "rm $COMPAREFILE;" echo "$1" > $COMPAREFILE shift - msgtest "$MSG" - $* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail -} -testequal() { - local EXPECTED="$1" - shift - local MSG="Test for equality of $*" - testequalwithmsg "$MSG" "$EXPECTED" $* + if [ -n "$MSG" ]; then + msgtest "$MSG" "$*" + fi + $* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail } testequalor2() { diff --git a/test/integration/test-apt-progress-fd-conffile b/test/integration/test-apt-progress-fd-conffile index 0b42b1b2f..085d5e871 100755 --- a/test/integration/test-apt-progress-fd-conffile +++ b/test/integration/test-apt-progress-fd-conffile @@ -32,12 +32,19 @@ testsuccess aptget install compiz-core=1.0 # fake conffile change echo "meep" >> rootdir/etc/compiz.conf/compiz.conf -# install +# FIXME: Is there really no way to see if dpkg actually prompts? +msgtest 'Test for successful execution of' 'apt-get install compiz-core=2.0' +OUTPUT=$(mktemp) +addtrap "rm $OUTPUT;" exec 3> apt-progress.log -echo n | aptget install compiz-core=2.0 -o APT::Status-Fd=3 -o Dpkg::Use-Pty=false +if aptget install compiz-core=2.0 -o APT::Status-Fd=3 -o Dpkg::Use-Pty=false -o dpkg::options::='--force-confold' >${OUTPUT} 2>&1; then + msgpass +else + echo + cat $OUTPUT + msgfail +fi # and ensure there is a conffile message in the file -msgtest "Conffile prompt in apt-progress.log" +msgtest 'Test status fd for an included' 'pmconffile msg' grep -q "pmconffile:/etc/compiz.conf/compiz.conf" apt-progress.log && msgpass || (cat apt-progress.log && msgfail) - -cat apt-progress.log \ No newline at end of file diff --git a/test/integration/test-apt-sources-deb822 b/test/integration/test-apt-sources-deb822 index bacad1ed4..8c022767f 100755 --- a/test/integration/test-apt-sources-deb822 +++ b/test/integration/test-apt-sources-deb822 @@ -5,17 +5,10 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' -SOURCES="rootdir/etc/apt/sources.list" - -echo "deb http://ftp.debian.org/debian stable main" > $SOURCES -testequalwithmsg "Old style sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : -'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris - - -BASE="# some comment +SOURCES='rootdir/etc/apt/sources.list' +BASE='# some comment # that contains a : as well #Type: meep @@ -24,57 +17,72 @@ URL: http://ftp.debian.org/debian Dist: stable Section: main Comment: Some random string - that can be very long" + that can be very long' -# simple case -echo "$BASE" > $SOURCES -testequalwithmsg "Simple deb822 sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : +msgtest 'Test old-style sources.list' +echo "deb http://ftp.debian.org/debian stable main" > $SOURCES +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : -'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris +'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris -# two sections (we support both "," and " " as seperator) -echo "$BASE" | sed s/main/"main,contrib"/ > $SOURCES +msgtest 'Test simple deb822 sources.list' +echo "$BASE" > $SOURCES +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris -testequalwithmsg "Two sections deb822 sources.list work" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : + +msgtest 'Test deb822 with two sections' 'seperated by comma' +echo "$BASE" | sed 's/main/main,contrib/' > $SOURCES +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/contrib/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_contrib_binary-i386_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/contrib/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_contrib_i18n_Translation-en 0 : 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : -'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris +'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris + +# FIXME: Advertised, but not supported at the moment +#msgtest 'Test deb822 with two sections' 'seperated by space' +#echo "$BASE" | sed 's/main/main contrib/' > $SOURCES +#testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : +#'http://ftp.debian.org/debian/dists/stable/contrib/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_contrib_binary-i386_Packages 0 : +#'http://ftp.debian.org/debian/dists/stable/contrib/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_contrib_i18n_Translation-en 0 : +#'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +#'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris -# Two entries + +msgtest 'Test deb822 with' 'two entries' echo "$BASE" > $SOURCES echo "" >> $SOURCES echo "$BASE" | sed s/stable/unstable/ >> $SOURCES - -testequalwithmsg "Multiple entries in deb822 sources.list work" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 : 'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 : -'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 " aptget update --print-uris +'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 " aptget update --print-uris -# ARCH option +msgtest 'Test deb822' 'architecture option' echo "$BASE" > $SOURCES echo "Arch: amd64,armel" >> $SOURCES - -testequalwithmsg "Arch: option in deb822 sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 : 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris -# invalid sources.list file -echo "deb http://ftp.debian.org" > $SOURCES -testequalwithmsg "Invalid sources.list file gives proper error" "E: Malformed line 1 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (dist) +msgtest 'Test old-style sources.list file which has' 'malformed dist' +echo "deb http://ftp.debian.org" > $SOURCES +testequal --nomsg "E: Malformed line 1 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (dist) E: The list of sources could not be read." aptget update --print-uris + +msgtest 'Test deb822 sources.list file which has' 'malformed URI' echo "Type: deb Dist: stable " > $SOURCES - -testequalwithmsg "Invalid deb822 sources.list file gives proper error" "E: Malformed stanza 0 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (URI parse) +testequal --nomsg "E: Malformed stanza 0 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (URI parse) E: The list of sources could not be read." aptget update --print-uris diff --git a/test/integration/test-bug-728500-tempdir b/test/integration/test-bug-728500-tempdir index 0606538a1..0451fc1ed 100755 --- a/test/integration/test-bug-728500-tempdir +++ b/test/integration/test-bug-728500-tempdir @@ -7,12 +7,23 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'i386' -buildsimplenativepackage 'coolstuff' 'all' '1.0' 'unstable' +insertpackage 'unstable' 'coolstuff' 'all' '1.0' setupaptarchive changetowebserver -msgtest 'Test with incorect TMPDIR' +msgtest 'Test apt-get update with incorrect' 'TMPDIR' + +OUTPUT=$(mktemp) +addtrap "rm $OUTPUT;" export TMPDIR=/does-not-exists -aptget update && msgpass || msgfail -unset TMPDIR \ No newline at end of file +if aptget update >${OUTPUT} 2>&1; then + msgpass +else + echo + cat $OUTPUT + msgfail +fi +unset TMPDIR + +testequal 'coolstuff' aptcache pkgnames -- cgit v1.2.3-70-g09d2 From 1410955589dc9f0eaa290907cac070b7ebf93b6a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 17 Jan 2014 08:43:14 +0100 Subject: add missing integration test for "apt list" --- apt-private/private-list.cc | 12 ++++++---- apt-private/private-output.cc | 8 ++++--- cmdline/apt.cc | 23 +++++++++++--------- test/integration/framework | 10 +++++---- test/integration/test-apt-binary | 47 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 21 deletions(-) create mode 100755 test/integration/test-apt-binary (limited to 'test') diff --git a/apt-private/private-list.cc b/apt-private/private-list.cc index a02ebf02d..fbb66d204 100644 --- a/apt-private/private-list.cc +++ b/apt-private/private-list.cc @@ -101,11 +101,15 @@ private: /*}}}*/ void ListAllVersions(pkgCacheFile &CacheFile, pkgRecords &records, /*{{{*/ pkgCache::PkgIterator P, - std::ostream &outs) + std::ostream &outs, + bool include_summary=true) { for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; Ver++) - ListSingleVersion(CacheFile, records, Ver, outs); + { + ListSingleVersion(CacheFile, records, Ver, outs, include_summary); + outs << "\n"; + } } /*}}}*/ // list - list package based on criteria /*{{{*/ @@ -136,7 +140,7 @@ bool List(CommandLine &Cmd) PackageNameMatcher matcher(patterns); LocalitySortedVersionSet bag; - OpTextProgress progress; + OpTextProgress progress(*_config); progress.OverallProgress(0, Cache->Head().PackageCount, Cache->Head().PackageCount, @@ -147,7 +151,7 @@ bool List(CommandLine &Cmd) std::stringstream outs; if(_config->FindB("APT::Cmd::All-Versions", false) == true) { - ListAllVersions(CacheFile, records, V.ParentPkg(), outs); + ListAllVersions(CacheFile, records, V.ParentPkg(), outs, includeSummary); output_map.insert(std::make_pair( V.ParentPkg().Name(), outs.str())); } else { diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index 91d13f31b..a8bbad9e5 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -114,11 +114,13 @@ std::string GetVersion(pkgCacheFile &CacheFile, pkgCache::VerIterator V)/*{{{*/ pkgCache::PkgIterator P = V.ParentPkg(); if (V == P.CurrentVer()) { + std::string inst_str = DeNull(V.VerStr()); +#if 0 // FIXME: do we want this or something like this? pkgDepCache *DepCache = CacheFile.GetDepCache(); pkgDepCache::StateCache &state = (*DepCache)[P]; - std::string inst_str = DeNull(V.VerStr()); if (state.Upgradable()) return "**"+inst_str; +#endif return inst_str; } @@ -224,11 +226,11 @@ void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, /*{{{*/ else out << GetVersion(CacheFile, V); } - out << " " << GetArchitecture(CacheFile, P) << " "; + out << " " << GetArchitecture(CacheFile, P); if (include_summary) { out << std::endl - << " " << GetShortDescription(CacheFile, records, P) + << " " << GetShortDescription(CacheFile, records, P) << std::endl; } } diff --git a/cmdline/apt.cc b/cmdline/apt.cc index 2d3966e86..8dc4c292a 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -114,16 +114,6 @@ int main(int argc, const char *argv[]) /*{{{*/ std::vector Args = getCommandArgs("apt", CommandLine::GetCommand(Cmds, argc, argv)); - if(!isatty(1)) - { - std::cerr << std::endl - << "WARNING: " << argv[0] << " " - << "does not have a stable CLI interface yet. " - << "Use with caution in scripts." - << std::endl - << std::endl; - } - InitOutput(); // Set up gettext support @@ -149,6 +139,19 @@ int main(int argc, const char *argv[]) /*{{{*/ return 100; } + if(!isatty(STDOUT_FILENO) && + _config->FindB("Apt::Cmd::Disable-Script-Warning", false) == false) + { + std::cerr << std::endl + << "WARNING: " << argv[0] << " " + << "does not have a stable CLI interface yet. " + << "Use with caution in scripts." + << std::endl + << std::endl; + } + if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) + _config->Set("quiet","1"); + // See if the help should be shown if (_config->FindB("help") == true || _config->FindB("version") == true || diff --git a/test/integration/framework b/test/integration/framework index a28363768..6620c78dd 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -99,6 +99,7 @@ aptconfig() { runapt apt-config $*; } aptcache() { runapt apt-cache $*; } aptcdrom() { runapt apt-cdrom $*; } aptget() { runapt apt-get $*; } +apt() { runapt apt $*; } aptftparchive() { runapt apt-ftparchive $*; } aptkey() { runapt apt-key $*; } aptmark() { runapt apt-mark $*; } @@ -202,6 +203,7 @@ setupenvironment() { echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf echo 'quiet::NoUpdate "true";' >> aptconfig.conf echo "Acquire::https::CaInfo \"${TESTDIR}/apt.pem\";" > rootdir/etc/apt/apt.conf.d/99https + echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary export LC_ALL=C export PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" configcompression '.' 'gz' #'bz2' 'lzma' 'xz' @@ -288,7 +290,7 @@ setupsimplenativepackage() { local VERSION="$3" local RELEASE="${4:-unstable}" local DEPENDENCIES="$5" - local DESCRIPTION="${6:-"Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} + local DESCRIPTION="${6:-"an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} If you find such a package installed on your system, something went horribly wrong! They are autogenerated und used only by testcases and surf no other propose…"}" @@ -338,7 +340,7 @@ buildsimplenativepackage() { local VERSION="$3" local RELEASE="${4:-unstable}" local DEPENDENCIES="$5" - local DESCRIPTION="${6:-"Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} + local DESCRIPTION="${6:-"an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} If you find such a package installed on your system, something went horribly wrong! They are autogenerated und used only by testcases and surf no other propose…"}" @@ -535,7 +537,7 @@ insertpackage() { local VERSION="$4" local DEPENDENCIES="$5" local PRIORITY="${6:-optional}" - local DESCRIPTION="${7:-"Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} + local DESCRIPTION="${7:-"an autogenerated dummy ${NAME}=${VERSION}/${RELEASE} If you find such a package installed on your system, something went horribly wrong! They are autogenerated und used only by testcases and surf no other propose…"}" @@ -595,7 +597,7 @@ insertinstalledpackage() { local DEPENDENCIES="$4" local PRIORITY="${5:-optional}" local STATUS="${6:-install ok installed}" - local DESCRIPTION="${7:-"Description: an autogenerated dummy ${NAME}=${VERSION}/installed + local DESCRIPTION="${7:-"an autogenerated dummy ${NAME}=${VERSION}/installed If you find such a package installed on your system, something went horribly wrong! They are autogenerated und used only by testcases and surf no other propose…"}" diff --git a/test/integration/test-apt-binary b/test/integration/test-apt-binary new file mode 100755 index 000000000..8d5df9051 --- /dev/null +++ b/test/integration/test-apt-binary @@ -0,0 +1,47 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '1.0' +insertinstalledpackage 'bar' 'i386' '1.0' + +insertinstalledpackage 'foobar' 'i386' '1.0' +insertpackage 'unstable' 'foobar' 'i386' '2.0' + +setupaptarchive + +APTARCHIVE=$(readlink -f ./aptarchive) + +testequal "Listing... +bar/now 1.0 [installed,local] i386 +foo/unstable 1.0 all +foobar/unstable 2.0 [upgradable from: 1.0] i386" apt list + +testequal "Listing... +foo/unstable 1.0 all +foobar/unstable 2.0 [upgradable from: 1.0] i386" apt list "foo*" + +testequal "Listing... +foobar/unstable 2.0 [upgradable from: 1.0] i386" apt list --upgradable + +# FIXME: hm, hm - does it make sense to have this different? shouldn't +# we use "installed,upgradable" consitently? +testequal "Listing... +bar/now 1.0 [installed,local] i386 +foobar/now 1.0 [installed,upgradable to: 2.0] i386" apt list --installed + +testequal "Listing... +foobar/unstable 2.0 [upgradable from: 1.0] i386 +foobar/now 1.0 [installed,upgradable to: 2.0] i386 +" apt list foobar --all-versions + +testequal "Listing... +bar/now 1.0 [installed,local] i386 + an autogenerated dummy bar=1.0/installed +" apt list bar --verbose + -- cgit v1.2.3-70-g09d2