diff options
author | Julian Andres Klode <jak@debian.org> | 2019-08-19 14:15:49 +0000 |
---|---|---|
committer | Julian Andres Klode <jak@debian.org> | 2019-08-19 14:15:49 +0000 |
commit | de951e5619f55c8281389a5c5986792f5453e602 (patch) | |
tree | fe9664d72c563e00e63e4acd83e6c1b21f8fa12c | |
parent | 7c724251fd8c24e89dc8cb813eee20aa0a4ad793 (diff) | |
parent | d18b6095862e8268b4d2cd8c0b3140829a1e4950 (diff) |
Merge branch 'pu/patterns' into 'master'
Package patterns
See merge request apt-team/apt!74
-rw-r--r-- | apt-pkg/cachefilter-patterns.cc | 306 | ||||
-rw-r--r-- | apt-pkg/cachefilter-patterns.h | 235 | ||||
-rw-r--r-- | apt-pkg/cachefilter.h | 4 | ||||
-rw-r--r-- | apt-pkg/cacheset.cc | 31 | ||||
-rw-r--r-- | apt-pkg/cacheset.h | 6 | ||||
-rw-r--r-- | apt-private/private-list.cc | 23 | ||||
-rw-r--r-- | debian/apt.install | 1 | ||||
-rw-r--r-- | doc/CMakeLists.txt | 1 | ||||
-rw-r--r-- | doc/apt-patterns.7.xml | 161 | ||||
-rw-r--r-- | doc/po4a.conf | 1 | ||||
-rwxr-xr-x | test/integration/test-apt-patterns | 173 | ||||
-rw-r--r-- | test/libapt/pattern_test.cc | 95 |
12 files changed, 1028 insertions, 9 deletions
diff --git a/apt-pkg/cachefilter-patterns.cc b/apt-pkg/cachefilter-patterns.cc new file mode 100644 index 000000000..bf6166ee4 --- /dev/null +++ b/apt-pkg/cachefilter-patterns.cc @@ -0,0 +1,306 @@ +/* + * cachefilter-patterns.cc - Parser for aptitude-style patterns + * + * Copyright (c) 2019 Canonical Ltd + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +#include <apt-pkg/cachefilter-patterns.h> + +namespace APT +{ +namespace Internal +{ + +template <class... Args> +std::string rstrprintf(Args... args) +{ + std::string str; + strprintf(str, std::forward<Args>(args)...); + return str; +} + +// Parse a complete pattern, make sure it's the entire input +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseTop() +{ + skipSpace(); + auto node = parse(); + skipSpace(); + + if (node->end != sentence.size()) + { + Node node2; + + node2.start = node->end; + node2.end = sentence.size(); + throw Error{node2, "Expected end of file"}; + } + + return node; +} + +// Parse any pattern +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parse() +{ + std::unique_ptr<Node> node; + if ((node = parsePattern()) != nullptr) + return node; + if ((node = parseQuotedWord()) != nullptr) + return node; + if ((node = parseWord()) != nullptr) + return node; + + Node eNode; + eNode.end = eNode.start = state.offset; + throw Error{eNode, "Expected pattern, quoted word, or word"}; +} + +// Parse a list pattern (or function call pattern) +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parsePattern() +{ + static const APT::StringView CHARS("0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "-"); + if (sentence[state.offset] != '?') + return nullptr; + + auto node = std::make_unique<PatternNode>(); + node->end = node->start = state.offset; + state.offset++; + + while (CHARS.find(sentence[state.offset]) != APT::StringView::npos) + { + ++state.offset; + } + + node->term = sentence.substr(node->start, state.offset - node->start); + + node->end = skipSpace(); + // We don't have any arguments, return node; + if (sentence[state.offset] != '(') + return node; + node->end = ++state.offset; + skipSpace(); + + node->haveArgumentList = true; + + // Empty argument list, return + if (sentence[state.offset] == ')') + { + node->end = ++state.offset; + return node; + } + + node->arguments.push_back(parse()); + skipSpace(); + while (sentence[state.offset] == ',') + { + ++state.offset; + skipSpace(); + // This was a trailing comma - allow it and break the loop + if (sentence[state.offset] == ')') + break; + node->arguments.push_back(parse()); + skipSpace(); + } + + node->end = state.offset; + if (sentence[state.offset] != ')') + throw Error{node->arguments.empty() ? *node : *node->arguments[node->arguments.size() - 1], + rstrprintf("Expected closing parenthesis or comma after last argument, received %c", sentence[state.offset])}; + + node->end = ++state.offset; + return node; +} + +// Parse a quoted word atom +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseQuotedWord() +{ + if (sentence[state.offset] != '"') + return nullptr; + + auto node = std::make_unique<WordNode>(); + node->start = state.offset; + + // Eat beginning of string + state.offset++; + + while (sentence[state.offset] != '"' && sentence[state.offset] != '\0') + state.offset++; + + // End of string + if (sentence[state.offset] != '"') + throw Error{*node, "Could not find end of string"}; + state.offset++; + + node->end = state.offset; + node->word = sentence.substr(node->start + 1, node->end - node->start - 2); + + return node; +} + +// Parse a bare word atom +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseWord() +{ + static const APT::StringView DISALLOWED_START("?~,()\0", 6); + static const APT::StringView DISALLOWED(",()\0", 4); + if (DISALLOWED_START.find(sentence[state.offset]) != APT::StringView::npos) + return nullptr; + + auto node = std::make_unique<WordNode>(); + node->start = state.offset; + + while (DISALLOWED.find(sentence[state.offset]) == APT::StringView::npos) + state.offset++; + + node->end = state.offset; + node->word = sentence.substr(node->start, node->end - node->start); + return node; +} + +// Rendering of the tree in JSON for debugging +std::ostream &PatternTreeParser::PatternNode::render(std::ostream &os) +{ + os << "{" + << "\"term\": \"" << term.to_string() << "\",\n" + << "\"arguments\": [\n"; + for (auto &node : arguments) + node->render(os) << "," << std::endl; + os << "null]\n"; + os << "}\n"; + return os; +} + +std::ostream &PatternTreeParser::WordNode::render(std::ostream &os) +{ + os << '"' << word.to_string() << '"'; + return os; +} + +std::nullptr_t PatternTreeParser::Node::error(std::string message) +{ + throw Error{*this, message}; +} + +bool PatternTreeParser::PatternNode::matches(APT::StringView name, int min, int max) +{ + if (name != term) + return false; + if (max != 0 && !haveArgumentList) + error(rstrprintf("%s expects an argument list", term.to_string().c_str())); + if (max == 0 && haveArgumentList) + error(rstrprintf("%s does not expect an argument list", term.to_string().c_str())); + if (min >= 0 && min == max && (arguments.size() != size_t(min))) + error(rstrprintf("%s expects %d arguments, but received %d arguments", term.to_string().c_str(), min, arguments.size())); + if (min >= 0 && arguments.size() < size_t(min)) + error(rstrprintf("%s expects at least %d arguments, but received %d arguments", term.to_string().c_str(), min, arguments.size())); + if (max >= 0 && arguments.size() > size_t(max)) + error(rstrprintf("%s expects at most %d arguments, but received %d arguments", term.to_string().c_str(), max, arguments.size())); + return true; +} + +std::unique_ptr<APT::CacheFilter::Matcher> PatternParser::aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP) +{ + assert(nodeP != nullptr); + auto node = dynamic_cast<PatternTreeParser::PatternNode *>(nodeP.get()); + if (node == nullptr) + nodeP->error("Expected a pattern"); + + if (node->matches("?architecture", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageArchitectureMatchesSpecification>(aWord(node->arguments[0])); + if (node->matches("?automatic", 0, 0)) + return std::make_unique<Patterns::PackageIsAutomatic>(file); + if (node->matches("?broken", 0, 0)) + return std::make_unique<Patterns::PackageIsBroken>(file); + if (node->matches("?config-files", 0, 0)) + return std::make_unique<Patterns::PackageIsConfigFiles>(); + if (node->matches("?essential", 0, 0)) + return std::make_unique<Patterns::PackageIsEssential>(); + if (node->matches("?exact-name", 1, 1)) + return std::make_unique<Patterns::PackageHasExactName>(aWord(node->arguments[0])); + if (node->matches("?false", 0, 0)) + return std::make_unique<APT::CacheFilter::FalseMatcher>(); + if (node->matches("?garbage", 0, 0)) + return std::make_unique<Patterns::PackageIsGarbage>(file); + if (node->matches("?installed", 0, 0)) + return std::make_unique<Patterns::PackageIsInstalled>(file); + if (node->matches("?name", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageNameMatchesRegEx>(aWord(node->arguments[0])); + if (node->matches("?not", 1, 1)) + return std::make_unique<APT::CacheFilter::NOTMatcher>(aPattern(node->arguments[0]).release()); + if (node->matches("?obsolete", 0, 0)) + return std::make_unique<Patterns::PackageIsObsolete>(); + if (node->matches("?true", 0, 0)) + return std::make_unique<APT::CacheFilter::TrueMatcher>(); + if (node->matches("?upgradable", 0, 0)) + return std::make_unique<Patterns::PackageIsUpgradable>(file); + if (node->matches("?virtual", 0, 0)) + return std::make_unique<Patterns::PackageIsVirtual>(); + if (node->matches("?x-name-fnmatch", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageNameMatchesFnmatch>(aWord(node->arguments[0])); + + // Variable argument patterns + if (node->matches("?and", 0, -1)) + { + auto pattern = std::make_unique<APT::CacheFilter::ANDMatcher>(); + for (auto &arg : node->arguments) + pattern->AND(aPattern(arg).release()); + return pattern; + } + if (node->matches("?or", 0, -1)) + { + auto pattern = std::make_unique<APT::CacheFilter::ORMatcher>(); + + for (auto &arg : node->arguments) + pattern->OR(aPattern(arg).release()); + return pattern; + } + + node->error(rstrprintf("Unrecognized pattern '%s'", node->term.to_string().c_str())); + + return nullptr; +} + +std::string PatternParser::aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP) +{ + assert(nodeP != nullptr); + auto node = dynamic_cast<PatternTreeParser::WordNode *>(nodeP.get()); + if (node == nullptr) + nodeP->error("Expected a word"); + return node->word.to_string(); +} + +} // namespace Internal + +// The bridge into the public world +std::unique_ptr<APT::CacheFilter::Matcher> APT::CacheFilter::ParsePattern(APT::StringView pattern, pkgCacheFile *file) +{ + if (file != nullptr && !file->BuildDepCache()) + return nullptr; + + try + { + auto top = APT::Internal::PatternTreeParser(pattern).parseTop(); + APT::Internal::PatternParser parser{file}; + return parser.aPattern(top); + } + catch (APT::Internal::PatternTreeParser::Error &e) + { + std::stringstream ss; + ss << "input:" << e.location.start << "-" << e.location.end << ": error: " << e.message << "\n"; + ss << pattern.to_string() << "\n"; + for (size_t i = 0; i < e.location.start; i++) + ss << " "; + for (size_t i = e.location.start; i < e.location.end; i++) + ss << "^"; + + ss << "\n"; + + _error->Error("%s", ss.str().c_str()); + return nullptr; + } +} + +} // namespace APT diff --git a/apt-pkg/cachefilter-patterns.h b/apt-pkg/cachefilter-patterns.h new file mode 100644 index 000000000..d37da815f --- /dev/null +++ b/apt-pkg/cachefilter-patterns.h @@ -0,0 +1,235 @@ +/* + * cachefilter-patterns.h - Pattern parser and additional patterns as matchers + * + * Copyright (c) 2019 Canonical Ltd + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +#ifndef APT_CACHEFILTER_PATTERNS_H +#define APT_CACHEFILTER_PATTERNS_H +#include <apt-pkg/cachefile.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/error.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <iostream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> +#include <assert.h> +namespace APT +{ + +namespace Internal +{ +/** + * \brief PatternTreeParser parses the given sentence into a parse tree. + * + * The parse tree consists of nodes: + * - Word nodes which contains words or quoted words + * - Patterns, which represent ?foo and ?foo(...) patterns + */ +struct PatternTreeParser +{ + + struct Node + { + size_t start = 0; + size_t end = 0; + + virtual std::ostream &render(std::ostream &os) { return os; }; + std::nullptr_t error(std::string message); + }; + + struct Error : public std::exception + { + Node location; + std::string message; + + Error(Node location, std::string message) : location(location), message(message) {} + const char *what() const throw() override { return message.c_str(); } + }; + + struct PatternNode : public Node + { + APT::StringView term; + std::vector<std::unique_ptr<Node>> arguments; + bool haveArgumentList = false; + + std::ostream &render(std::ostream &stream) override; + bool matches(APT::StringView name, int min, int max); + }; + + struct WordNode : public Node + { + APT::StringView word; + bool quoted = false; + std::ostream &render(std::ostream &stream) override; + }; + + struct State + { + off_t offset = 0; + }; + + APT::StringView sentence; + State state; + + PatternTreeParser(APT::StringView sentence) : sentence(sentence){}; + off_t skipSpace() + { + while (sentence[state.offset] == ' ' || sentence[state.offset] == '\t' || sentence[state.offset] == '\r' || sentence[state.offset] == '\n') + state.offset++; + return state.offset; + }; + + /// \brief Parse a complete pattern + /// + /// There may not be anything before or after the pattern, except for + /// whitespace. + std::unique_ptr<Node> parseTop(); + + private: + std::unique_ptr<Node> parse(); + std::unique_ptr<Node> parsePattern(); + std::unique_ptr<Node> parseWord(); + std::unique_ptr<Node> parseQuotedWord(); +}; + +/** + * \brief PatternParser parses the given sentence into a parse tree. + * + * The parse tree consists of nodes: + * - Word nodes which contains words or quoted words + * - Patterns, which represent ?foo and ?foo(...) patterns + */ +struct PatternParser +{ + pkgCacheFile *file; + + std::unique_ptr<APT::CacheFilter::Matcher> aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP); + std::string aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP); +}; + +namespace Patterns +{ +using namespace APT::CacheFilter; + +struct PackageIsAutomatic : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsAutomatic(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return ((*Cache)[Pkg].Flags & pkgCache::Flag::Auto) != 0; + } +}; + +struct PackageIsBroken : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsBroken(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + auto state = (*Cache)[Pkg]; + return state.InstBroken() || state.NowBroken(); + } +}; + +struct PackageIsConfigFiles : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg->CurrentState == pkgCache::State::ConfigFiles; + } +}; + +struct PackageIsGarbage : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsGarbage(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return (*Cache)[Pkg].Garbage; + } +}; +struct PackageIsEssential : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return (Pkg->Flags & pkgCache::Flag::Essential) != 0; + } +}; + +struct PackageHasExactName : public PackageMatcher +{ + std::string name; + explicit PackageHasExactName(std::string name) : name(name) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg.Name() == name; + } +}; + +struct PackageIsInstalled : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsInstalled(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return Pkg->CurrentVer != 0; + } +}; + +struct PackageIsObsolete : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &pkg) override + { + // This code can be written without loops, as aptitude does, but it + // is far less readable. + if (pkg.CurrentVer().end()) + return false; + + // See if there is any version that exists in a repository, + // if so return false + for (auto ver = pkg.VersionList(); !ver.end(); ver++) + { + for (auto file = ver.FileList(); !file.end(); file++) + { + if ((file.File()->Flags & pkgCache::Flag::NotSource) == 0) + return false; + } + } + + return true; + } +}; + +struct PackageIsUpgradable : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsUpgradable(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return Pkg->CurrentVer != 0 && (*Cache)[Pkg].Upgradable(); + } +}; + +struct PackageIsVirtual : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg->VersionList == 0; + } +}; +} // namespace Patterns +} // namespace Internal +} // namespace APT +#endif diff --git a/apt-pkg/cachefilter.h b/apt-pkg/cachefilter.h index 8a6c01341..3c6e1559d 100644 --- a/apt-pkg/cachefilter.h +++ b/apt-pkg/cachefilter.h @@ -7,7 +7,9 @@ #define APT_CACHEFILTER_H // Include Files /*{{{*/ #include <apt-pkg/pkgcache.h> +#include <apt-pkg/string_view.h> +#include <memory> #include <string> #include <vector> @@ -145,6 +147,8 @@ public: }; /*}}}*/ +/// \brief Parse a pattern, return nullptr or pattern +std::unique_ptr<APT::CacheFilter::Matcher> ParsePattern(APT::StringView pattern, pkgCacheFile *file); } } #endif diff --git a/apt-pkg/cacheset.cc b/apt-pkg/cacheset.cc index 789727266..dd55edb4e 100644 --- a/apt-pkg/cacheset.cc +++ b/apt-pkg/cacheset.cc @@ -46,6 +46,7 @@ bool CacheSetHelper::PackageFrom(enum PkgSelector const select, PackageContainer case FNMATCH: return PackageFromFnmatch(pci, Cache, pattern); case PACKAGENAME: return PackageFromPackageName(pci, Cache, pattern); case STRING: return PackageFromString(pci, Cache, pattern); + case PATTERN: return PackageFromPattern(pci, Cache, pattern); } return false; } @@ -281,13 +282,33 @@ bool CacheSetHelper::PackageFromPackageName(PackageContainerInterface * const pc pci->insert(Pkg); return true; } + +bool CacheSetHelper::PackageFromPattern(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string const &pattern) +{ + if (pattern.size() < 1 || pattern[0] != '?') + return false; + + auto compiledPattern = APT::CacheFilter::ParsePattern(pattern, &Cache); + if (!compiledPattern) + return false; + + for (pkgCache::PkgIterator Pkg = Cache->PkgBegin(); Pkg.end() == false; ++Pkg) + { + if ((*compiledPattern)(Pkg) == false) + continue; + + pci->insert(Pkg); + } + return true; +} /*}}}*/ // PackageFromString - Return all packages matching a specific string /*{{{*/ bool CacheSetHelper::PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &str) { bool found = true; _error->PushToStack(); - if (PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str) == false && + if (PackageFrom(CacheSetHelper::PATTERN, pci, Cache, str) == false && + PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str) == false && PackageFrom(CacheSetHelper::TASK, pci, Cache, str) == false && // FIXME: hm, hm, regexp/fnmatch incompatible? PackageFrom(CacheSetHelper::FNMATCH, pci, Cache, str) == false && @@ -686,6 +707,7 @@ void CacheSetHelper::canNotFindPackage(enum PkgSelector const select, case FNMATCH: canNotFindFnmatch(pci, Cache, pattern); break; case PACKAGENAME: canNotFindPackage(pci, Cache, pattern); break; case STRING: canNotFindPackage(pci, Cache, pattern); break; + case PATTERN: canNotFindPackage(pci, Cache, pattern); break; case UNKNOWN: break; } } @@ -822,6 +844,7 @@ void CacheSetHelper::showPackageSelection(pkgCache::PkgIterator const &pkg, enum case REGEX: showRegExSelection(pkg, pattern); break; case TASK: showTaskSelection(pkg, pattern); break; case FNMATCH: showFnmatchSelection(pkg, pattern); break; + case PATTERN: showPatternSelection(pkg, pattern); break; case PACKAGENAME: /* no surprises here */ break; case STRING: /* handled by the special cases */ break; case UNKNOWN: break; @@ -842,6 +865,12 @@ void CacheSetHelper::showFnmatchSelection(pkgCache::PkgIterator const &/*pkg*/, std::string const &/*pattern*/) { } /*}}}*/ +// showPatternSelection /*{{{*/ +void CacheSetHelper::showPatternSelection(pkgCache::PkgIterator const & /*pkg*/, + std::string const & /*pattern*/) +{ +} + /*}}}*/ /*}}}*/ // showVersionSelection /*{{{*/ void CacheSetHelper::showVersionSelection(pkgCache::PkgIterator const &Pkg, diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h index 489fb6220..6023b861d 100644 --- a/apt-pkg/cacheset.h +++ b/apt-pkg/cacheset.h @@ -52,7 +52,7 @@ public: /*{{{*/ GlobalError::MsgType ErrorType = GlobalError::ERROR); virtual ~CacheSetHelper(); - enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING }; + enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING, PATTERN }; virtual bool PackageFrom(enum PkgSelector const select, PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); @@ -172,10 +172,12 @@ protected: bool PackageFromFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); bool PackageFromPackageName(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); bool PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); + bool PackageFromPattern(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); private: void showTaskSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); void showRegExSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); void showFnmatchSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + void showPatternSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); void canNotFindTask(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); void canNotFindRegEx(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); void canNotFindFnmatch(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); @@ -739,6 +741,8 @@ public: std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper, bool const onlyFromName = false); + static bool FromPattern(VersionContainerInterface *const vci, pkgCacheFile &Cache, + std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper); static bool FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &P, CacheSetHelper::VerSelector const fallback, diff --git a/apt-private/private-list.cc b/apt-private/private-list.cc index 7c8c89777..6071129a7 100644 --- a/apt-private/private-list.cc +++ b/apt-private/private-list.cc @@ -39,17 +39,26 @@ struct PackageSortAlphabetic /*{{{*/ class PackageNameMatcher : public Matcher { + pkgCacheFile &cacheFile; public: - explicit PackageNameMatcher(const char **patterns) + explicit PackageNameMatcher(pkgCacheFile &cacheFile, const char **patterns) + : cacheFile(cacheFile) { for(int i=0; patterns[i] != NULL; ++i) { std::string pattern = patterns[i]; - APT::CacheFilter::PackageMatcher *cachefilter = NULL; - if(_config->FindB("APT::Cmd::Use-Regexp", false) == true) + APT::CacheFilter::Matcher *cachefilter = NULL; + if (pattern.size() > 0 && pattern[0] == '?') + cachefilter = APT::CacheFilter::ParsePattern(pattern, &cacheFile).release(); + else if(_config->FindB("APT::Cmd::Use-Regexp", false) == true) cachefilter = new APT::CacheFilter::PackageNameMatchesRegEx(pattern); else cachefilter = new APT::CacheFilter::PackageNameMatchesFnmatch(pattern); + + if (cachefilter == nullptr) { + return; + filters.clear(); + } filters.push_back(cachefilter); } } @@ -62,7 +71,7 @@ class PackageNameMatcher : public Matcher { for(J=filters.begin(); J != filters.end(); ++J) { - APT::CacheFilter::PackageMatcher *cachefilter = *J; + APT::CacheFilter::Matcher *cachefilter = *J; if((*cachefilter)(P)) return true; } @@ -70,8 +79,8 @@ class PackageNameMatcher : public Matcher } private: - std::vector<APT::CacheFilter::PackageMatcher*> filters; - std::vector<APT::CacheFilter::PackageMatcher*>::const_iterator J; + std::vector<APT::CacheFilter::Matcher*> filters; + std::vector<APT::CacheFilter::Matcher*>::const_iterator J; #undef PackageMatcher }; /*}}}*/ @@ -111,7 +120,7 @@ bool DoList(CommandLine &Cmd) if (_config->FindB("APT::Cmd::List-Include-Summary", false) == true) format += "\n ${Description}\n"; - PackageNameMatcher matcher(patterns); + PackageNameMatcher matcher(CacheFile, patterns); LocalitySortedVersionSet bag; OpTextProgress progress(*_config); progress.OverallProgress(0, diff --git a/debian/apt.install b/debian/apt.install index f12eb240d..f745b3de5 100644 --- a/debian/apt.install +++ b/debian/apt.install @@ -37,6 +37,7 @@ usr/share/man/*/apt-get.* usr/share/man/*/apt-key.* usr/share/man/*/apt-mark.* usr/share/man/*/apt-secure.* +usr/share/man/*/apt-patterns.* usr/share/man/*/apt-transport-http.* usr/share/man/*/apt-transport-https.* usr/share/man/*/apt-transport-mirror.* diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7cca4cf81..3060949e5 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -82,6 +82,7 @@ add_docbook(apt-man MANPAGE ALL apt-key.8.xml apt-mark.8.xml apt_preferences.5.xml + apt-patterns.7.xml apt-secure.8.xml apt-sortpkgs.1.xml apt-transport-http.1.xml diff --git a/doc/apt-patterns.7.xml b/doc/apt-patterns.7.xml new file mode 100644 index 000000000..efd4293dc --- /dev/null +++ b/doc/apt-patterns.7.xml @@ -0,0 +1,161 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ +<!ENTITY % aptent SYSTEM "apt.ent"> %aptent; +<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment; +<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor; +]> + +<refentry> + <refentryinfo> + &apt-author.jgunthorpe; + &apt-author.team; + &apt-email; + &apt-product; + <!-- The last update date --> + <date>2019-08-15T00:00:00Z</date> + </refentryinfo> + + <refmeta> + <refentrytitle>apt-patterns</refentrytitle> + <manvolnum>7</manvolnum> + <refmiscinfo class="manual">APT</refmiscinfo> + </refmeta> + + <!-- Man page title --> + <refnamediv> + <refname>apt-patterns</refname> + <refpurpose>Syntax and semantics of apt search patterns</refpurpose> + </refnamediv> + + <refsect1><title>Description</title> + <para> + Starting with version 2.0, <command>APT</command> provides support for + patterns, which can be used to query the apt cache for packages. + </para> + </refsect1> + + <refsect1> + <title>Logic patterns</title> + <para> + These patterns provide the basic means to combine other patterns into + more complex expressions, as well as <code>?true</code> and <code>?false</code> + patterns. + </para> + <variablelist> + <varlistentry><term><code>?and(PATTERN, PATTERN, ...)</code></term> + <listitem><para>Selects objects where all specified patterns match.</para></listitem> + </varlistentry> + <varlistentry><term><code>?false</code></term> + <listitem><para>Selects nothing.</para></listitem> + </varlistentry> + <varlistentry><term><code>?not(PATTERN)</code></term> + <listitem><para>Selects objects where PATTERN does not match.</para></listitem> + </varlistentry> + <varlistentry><term><code>?or(PATTERN, PATTERN, ...)</code></term> + <listitem><para>Selects objects where at least one of the specified patterns match.</para></listitem> + </varlistentry> + <varlistentry><term><code>?true</code></term> + <listitem><para>Selects all objects.</para></listitem> + </varlistentry> + </variablelist> + </refsect1> + <refsect1> + <title>Package patterns</title> + <para> + These patterns select specific packages. + </para> + <variablelist> + <varlistentry><term><code>?architecture(WILDCARD)</code></term> + <listitem><para>Selects packages matching the specified architecture, which may contain wildcards using any.</para></listitem> + </varlistentry> + <varlistentry><term><code>?automatic</code></term> + <listitem><para>Selects packages that were installed automatically.</para></listitem> + </varlistentry> + <varlistentry><term><code>?broken</code></term> + <listitem><para>Selects packages that have broken dependencies.</para></listitem> + </varlistentry> + <varlistentry><term><code>?config-files</code></term> + <listitem><para>Selects packages that are not fully installed, but have solely residual configuration files left.</para></listitem> + </varlistentry> + <varlistentry><term><code>?essential</code></term> + <listitem><para>Selects packages that have Essential: yes set in their control file.</para></listitem> + </varlistentry> + <varlistentry><term><code>?exact-name(NAME)</code></term> + <listitem><para>Selects packages with the exact specified name.</para></listitem> + </varlistentry> + <varlistentry><term><code>?garbage</code></term> + <listitem><para>Selects packages that can be removed automatically.</para></listitem> + </varlistentry> + <varlistentry><term><code>?installed</code></term> + <listitem><para>Selects packages that are currently installed.</para></listitem> + </varlistentry> + <varlistentry><term><code>?name(REGEX)</code></term> + <listitem><para>Selects packages where the name matches the given regular expression.</para></listitem> + </varlistentry> + <varlistentry><term><code>?obsolete</code></term> + <listitem><para>Selects packages that no longer exist in repositories.</para></listitem> + </varlistentry> + <varlistentry><term><code>?upgradable</code></term> + <listitem><para>Selects packages that can be upgraded (have a newer candidate).</para></listitem> + </varlistentry> + <varlistentry><term><code>?virtual</code></term> + <listitem><para>Selects all virtual packages; that is packages without a version. + These exist when they are referenced somewhere in the archive, + for example because something depends on that name.</para></listitem> + </varlistentry> + </variablelist> + </refsect1> + + + <refsect1><title>Examples</title> + <variablelist> + <varlistentry><term><code>apt remove ?garbage</code></term> + <listitem><para>Remove all packages that are automatically installed and no longer needed - same as apt autoremove</para></listitem> + </varlistentry> + <varlistentry><term><code>apt purge ?config-files</code></term> + <listitem><para>Purge all packages that only have configuration files left</para></listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1><title>Migrating from aptitude</title> + <para> + Patterns in apt are heavily inspired by patterns in aptitude, but with some tweaks: + </para> + <itemizedlist> + <listitem> + <para>Only long forms — the ones starting with ? — are supported</para> + </listitem> + <listitem> + <para> + Syntax is uniform: If there is an opening parenthesis after a term, it is always assumed to be the beginning of an argument list. + </para> + <para> + In aptitude, a syntactic form <code>"?foo(bar)"</code> could mean <code>"?and(?foo,bar)"</code> if foo does not take an argument. In APT, this will cause an error. + </para> + </listitem> + <listitem> + <para>Not all patterns are supported.</para> + </listitem> + <listitem> + <para>Some additional patterns are available, for example, for finding gstreamer codecs.</para> + </listitem> + <listitem> + <para>Escaping terms with <code>~</code> is not supported.</para> + </listitem> + <listitem> + <para>A trailing comma is allowed in argument lists</para> + </listitem> + </itemizedlist> + </refsect1> + + <refsect1><title>See Also</title> + <para> + &apt-get;, &apt; + </para> + </refsect1> + + &manbugs; + &manauthor; +</refentry> diff --git a/doc/po4a.conf b/doc/po4a.conf index 587215abc..1cf170b80 100644 --- a/doc/po4a.conf +++ b/doc/po4a.conf @@ -30,6 +30,7 @@ [type: manpage] apt-transport-http.1.xml $lang:$lang/apt-transport-http.$lang.1.xml add_$lang:xml.add [type: manpage] apt-transport-https.1.xml $lang:$lang/apt-transport-https.$lang.1.xml add_$lang:xml.add [type: manpage] apt-transport-mirror.1.xml $lang:$lang/apt-transport-mirror.$lang.1.xml add_$lang:xml.add +[type: manpage] apt-patterns.7.xml $lang:$lang/apt-patterns.7.xml add_$lang:xml.add [type: docbook] guide.dbk $lang:$lang/guide.$lang.dbk # add_$lang::$lang/addendum/docbook_$lang.add diff --git a/test/integration/test-apt-patterns b/test/integration/test-apt-patterns new file mode 100755 index 000000000..92c76edd1 --- /dev/null +++ b/test/integration/test-apt-patterns @@ -0,0 +1,173 @@ +#!/bin/sh +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture 'i386' 'amd64' + +insertpackage 'unstable' 'available' 'all' '1.0' + +insertinstalledpackage 'manual1' 'i386' '1.0' 'Depends: automatic1' +insertinstalledpackage 'manual2' 'i386' '1.0' + +insertinstalledpackage 'automatic1' 'i386' '1.0' +insertinstalledpackage 'automatic2' 'i386' '1.0' + +insertinstalledpackage 'essential' 'i386' '1.0' 'Essential: yes' +insertinstalledpackage 'conf-only' 'i386' '1.0' '' '' 'deinstall ok config-files' +insertinstalledpackage 'broken' 'i386' '1.0' 'Depends: does-not-exist' + +insertinstalledpackage 'not-obsolete' 'i386' '1.0' +insertpackage 'unstable' 'not-obsolete' 'all' '2.0' + +insertpackage 'unstable' 'foreign' 'amd64' '2.0' + +setupaptarchive + +testsuccess aptmark auto automatic1 automatic2 + +msgmsg "Check that commands understand patterns" + +testfailureequal "E: input:0-14: error: Unrecognized pattern '?not-a-pattern' + ?not-a-pattern + ^^^^^^^^^^^^^^ +N: Unable to locate package ?not-a-pattern +N: Couldn't find any package by glob '?not-a-pattern' +E: Regex compilation error - Invalid preceding regular expression +N: Couldn't find any package by regex '?not-a-pattern' +E: input:0-14: error: Unrecognized pattern '?not-a-pattern' + ?not-a-pattern + ^^^^^^^^^^^^^^ +N: Unable to locate package ?not-a-pattern +N: Couldn't find any package by glob '?not-a-pattern' +E: Regex compilation error - Invalid preceding regular expression +N: Couldn't find any package by regex '?not-a-pattern' +E: No packages found" apt show '?not-a-pattern' + +testfailureequal "Listing... +E: input:0-14: error: Unrecognized pattern '?not-a-pattern' + ?not-a-pattern + ^^^^^^^^^^^^^^" apt list '?not-a-pattern' + +testfailureequal "Reading package lists... +Building dependency tree... +Reading state information... +E: input:0-14: error: Unrecognized pattern '?not-a-pattern' + ?not-a-pattern + ^^^^^^^^^^^^^^ +E: Unable to locate package ?not-a-pattern +E: Couldn't find any package by glob '?not-a-pattern' +E: Regex compilation error - Invalid preceding regular expression +E: Couldn't find any package by regex '?not-a-pattern'" apt install -s '?not-a-pattern' + + +msgmsg "Ensure that argument lists are present where needed, and absent elsewhere" + +testfailureequal "Listing... +E: input:0-7: error: ?true does not expect an argument list + ?true() + ^^^^^^^" apt list '?true()' +testfailureequal "Listing... +E: input:0-4: error: ?and expects an argument list + ?and + ^^^^" apt list '?and' +testfailureequal "Listing... +E: input:0-3: error: ?or expects an argument list + ?or + ^^^" apt list '?or' + + +msgmsg "Basic logic: true, false, not, ?or, ?and" +for pattern in '?true' '?not(?false)'; do +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +automatic2/now 1.0 i386 [installed,local] +available/unstable 1.0 all +broken/now 1.0 i386 [installed,local] +conf-only/now 1.0 i386 [residual-config] +dpkg/now 1.16.2+fake all [installed,local] +essential/now 1.0 i386 [installed,local] +foreign/unstable 2.0 amd64 +manual1/now 1.0 i386 [installed,local] +manual2/now 1.0 i386 [installed,local] +not-obsolete/unstable 2.0 i386 [upgradable from: 1.0]" apt list "$pattern" +done +testsuccessequal "Listing..." apt list '?false' +testsuccessequal "Listing..." apt list '?not(?true)' +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +automatic2/now 1.0 i386 [installed,local] +manual1/now 1.0 i386 [installed,local] +manual2/now 1.0 i386 [installed,local]" apt list '?or(?name(^automatic),?name(^manual))' +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local]" apt list '?and(?name(^automatic),?name(1$))' + + +msgmsg "Package patterns" + +testsuccessequal "Listing... +foreign/unstable 2.0 amd64" apt list '?architecture(amd64)' + +# XXX FIXME We should have support for foreign and native +testsuccessequal "Listing..." apt list '?architecture(foreign)' +testsuccessequal "Listing..." apt list '?architecture(native)' + +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +automatic2/now 1.0 i386 [installed,local]" apt list '?automatic' + +testsuccessequal "Listing... +broken/now 1.0 i386 [installed,local]" apt list '?broken' + +testsuccessequal "Listing... +conf-only/now 1.0 i386 [residual-config]" apt list '?config-files' + +testsuccessequal "Listing... +essential/now 1.0 i386 [installed,local]" apt list '?essential' + +testsuccessequal "Listing..." apt list '?exact-name(automatic)' +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local]" apt list '?exact-name(automatic1)' + +testsuccessequal "Listing... +automatic2/now 1.0 i386 [installed,local]" apt list '?garbage' + +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +automatic2/now 1.0 i386 [installed,local] +broken/now 1.0 i386 [installed,local] +dpkg/now 1.16.2+fake all [installed,local] +essential/now 1.0 i386 [installed,local] +manual1/now 1.0 i386 [installed,local] +manual2/now 1.0 i386 [installed,local] +not-obsolete/unstable 2.0 i386 [upgradable from: 1.0]" apt list '?installed' + +testsuccessequal "Listing... +available/unstable 1.0 all +conf-only/now 1.0 i386 [residual-config] +foreign/unstable 2.0 amd64" apt list '?not(?installed)' + +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +automatic2/now 1.0 i386 [installed,local]" apt list '?name(^automatic)' + +testsuccessequal "Listing... +available/unstable 1.0 all +conf-only/now 1.0 i386 [residual-config] +foreign/unstable 2.0 amd64 +not-obsolete/unstable 2.0 i386 [upgradable from: 1.0]" apt list '?not(?obsolete)' + +testsuccessequal "Listing... +not-obsolete/unstable 2.0 i386 [upgradable from: 1.0] +N: There is 1 additional version. Please use the '-a' switch to see it" apt list '?upgradable' + +testsuccessequal "Package: does-not-exist +State: not a real package (virtual) +N: Can't select candidate version from package does-not-exist as it has no candidate +N: Can't select versions from package 'does-not-exist' as it is purely virtual +N: No packages found" apt show '?virtual' + +testsuccessequal "Listing..." apt list '?x-name-fnmatch(1)' +testsuccessequal "Listing... +automatic1/now 1.0 i386 [installed,local] +manual1/now 1.0 i386 [installed,local]" apt list '?x-name-fnmatch(*1)' diff --git a/test/libapt/pattern_test.cc b/test/libapt/pattern_test.cc new file mode 100644 index 000000000..de2fbceb9 --- /dev/null +++ b/test/libapt/pattern_test.cc @@ -0,0 +1,95 @@ +/* + * cachefilter-patterns.h - Pattern parser and additional patterns as matchers + * + * Copyright (c) 2019 Canonical Ltd + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +#include <config.h> +#include <apt-pkg/cachefilter-patterns.h> +#include <apt-pkg/cachefilter.h> + +#include <gtest/gtest.h> + +using namespace APT::Internal; + +TEST(TreeParserTest, ParseWord) +{ + auto node = PatternTreeParser("word").parseTop(); + auto wordNode = dynamic_cast<PatternTreeParser::WordNode *>(node.get()); + + EXPECT_EQ(node.get(), wordNode); + EXPECT_EQ(wordNode->word, "word"); +} + +TEST(TreeParserTest, ParseQuotedWord) +{ + auto node = PatternTreeParser("\"a word\"").parseTop(); + auto wordNode = dynamic_cast<PatternTreeParser::WordNode *>(node.get()); + + EXPECT_EQ(node.get(), wordNode); + EXPECT_EQ(wordNode->word, "a word"); +} + +TEST(TreeParserTest, ParsePattern) +{ + auto node = PatternTreeParser("?hello").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_TRUE(patternNode->arguments.empty()); + EXPECT_FALSE(patternNode->haveArgumentList); +} + +TEST(TreeParserTest, ParseWithEmptyArgs) +{ + auto node = PatternTreeParser("?hello()").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_TRUE(patternNode->arguments.empty()); + EXPECT_TRUE(patternNode->haveArgumentList); +} + +TEST(TreeParserTest, ParseWithOneArgs) +{ + auto node = PatternTreeParser("?hello(foo)").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_EQ(1u, patternNode->arguments.size()); +} + +TEST(TreeParserTest, ParseWithManyArgs) +{ + auto node = PatternTreeParser("?hello(foo,bar)").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_EQ(2u, patternNode->arguments.size()); +} + +TEST(TreeParserTest, ParseWithManyArgsWithSpaces) +{ + auto node = PatternTreeParser("?hello (foo, bar)").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_EQ(2u, patternNode->arguments.size()); +} + +TEST(TreeParserTest, ParseWithManyArgsWithSpacesWithTrailingComma) +{ + auto node = PatternTreeParser("?hello (foo, bar,)").parseTop(); + auto patternNode = dynamic_cast<PatternTreeParser::PatternNode *>(node.get()); + + EXPECT_EQ(node.get(), patternNode); + EXPECT_EQ(patternNode->term, "?hello"); + EXPECT_EQ(2u, patternNode->arguments.size()); +} |