diff --git a/CHANGELOG.md b/CHANGELOG.md index a4009b90..aa4a7779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Added - script: expose `all_headers` to templates — the fine-grained, transitive, deduplicated set of header files Slang resolved via `` `include `` for the kept trees. Populated whenever the Slang pass runs (`--top`/`--trim-incdirs`/`--broken`/`--encrypted`). +- manifest: allow packages to re-export the `export_include_dirs` of their direct dependencies with `- dep: ` and an optional `dir: `, so that include directories can be passed on transitively (https://github.com/pulp-platform/bender/issues/340). ### Changed - `bender clone`: primary flag for the checkout directory is now `--working-dir`, matching `bender snapshot`'s flag for the same concept; `-p`/`--path` are kept as hidden aliases for backwards compatibility. diff --git a/book/src/manifest.md b/book/src/manifest.md index c42b7ab5..3ff75368 100644 --- a/book/src/manifest.md +++ b/book/src/manifest.md @@ -67,6 +67,16 @@ export_include_dirs: - include ``` +Entries can also re-export the include directories of a direct dependency, so that packages depending on this one see them as well. With only `dep`, all `export_include_dirs` of that dependency are passed on; with `dir`, only the one that exactly matches an entry of the dependency's `export_include_dirs`. See [Exported Include Directories](sources.md#exported-include-directories). + +```yaml +export_include_dirs: + - include + - dep: common_cells + dir: include + - dep: reqrsp_interface +``` + Setting `frozen` to `true` prevents Bender from updating dependencies beyond what is recorded in the lockfile. This is useful for chip packages in tape-out mode where dependency changes would require disastrous amounts of re-verification. ```yaml diff --git a/book/src/sources.md b/book/src/sources.md index 533d0715..17f6dae6 100644 --- a/book/src/sources.md +++ b/book/src/sources.md @@ -113,6 +113,28 @@ export_include_dirs: Any package that depends on `my_utils` will automatically have the `include` directory added to its include search path. +Only the packages that directly depend on `my_utils` receive its include directories. A package can pass the include directories of its dependencies on by listing the dependency with `dep`: + +```yaml +package: + name: my_bus + +dependencies: + common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: 1.39.0 } + reqrsp_interface: { git: "https://github.com/pulp-platform/reqrsp_interface.git", version: 0.4.0 } + +export_include_dirs: + - include/normal/dir + - dep: common_cells + dir: include # must match an existing export_include_dir of common_cells + - dep: reqrsp_interface # without `dir`, all export_include_dirs of the dependency are passed on +``` + +- `dep` must name a direct dependency of the package. +- `dir` is given relative to the root of that dependency and must exactly match one of its `export_include_dirs` (a leading `./` and trailing `/` are ignored); otherwise Bender reports an error. Absolute paths and `..` never match. +- Re-exports are resolved transitively: if `common_cells` itself re-exports directories of one of its dependencies, these are passed on as well. +- An optional `target` restricts a re-export, just like for plain entries. Targets of the re-exported directories in the dependency are preserved. + > **Best Practice:** To avoid naming collisions, place your headers in a sub-folder named after your package. > > **The Problem:** If two packages (`axi` and `apb`) both export a header named `typedefs.svh`, a file including `` `include "typedefs.svh" `` will be ambiguous, and the compiler will pick whichever directory it finds first in the include path. diff --git a/src/config.rs b/src/config.rs index b150f659..8595da5d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -45,6 +45,8 @@ pub struct Manifest { pub sources: Option, /// The include directories exported to dependent packages. pub export_include_dirs: Vec<(TargetSpec, PathBuf)>, + /// The include directories of dependencies re-exported to dependent packages. + pub reexport_include_dirs: Vec, /// The plugin binaries. pub plugins: IndexMap, /// Whether the dependencies of the manifest are frozen. @@ -80,6 +82,21 @@ impl PrefixPaths for Manifest { } } +/// A re-exported include directory of a dependency. +/// +/// Makes the exported include directories of a direct dependency available to +/// the packages depending on this one. +#[derive(Clone, Debug)] +pub struct ReexportIncludeDir { + /// The name of the dependency whose include directories are re-exported. + pub dep: String, + /// The directory of the dependency to re-export, as listed in its + /// `export_include_dirs`. If `None`, all of its directories are re-exported. + pub dir: Option, + /// The target spec for which the re-export applies. + pub target: TargetSpec, +} + /// A package definition. /// /// Contains the metadata for an individual package. @@ -446,7 +463,7 @@ pub struct PartialManifest { /// The source files. pub sources: Option>, /// The include directories exported to dependent packages. - pub export_include_dirs: Option>>, + pub export_include_dirs: Option>>, /// The plugin binaries. pub plugins: Option>, /// Whether the dependencies of the manifest are frozen. @@ -581,11 +598,29 @@ impl Validate for PartialManifest { ), None => None, }; - let exp_inc_dirs = self + let mut exp_inc_dirs = Vec::new(); + let mut reexp_inc_dirs = Vec::new(); + for entry in self .export_include_dirs .unwrap_or_default() .validate(vctx) - .wrap_err(format!("In export_include_dirs of package `{}`:", pkg.name))?; + .wrap_err(format!("In export_include_dirs of package `{}`:", pkg.name))? + { + match entry { + ExportIncludeDir::Local(trgt, path) => exp_inc_dirs.push((trgt, path)), + ExportIncludeDir::Dep(reexport) => { + if deps.contains_key(&reexport.dep) { + reexp_inc_dirs.push(reexport); + } else { + Errors::ExportDirUnknownDep { + pkg: pkg.name.clone(), + dep: reexport.dep, + } + .downgrade_if_suppressed()?; + } + } + } + } let plugins = match self.plugins { Some(s) => s .iter() @@ -639,6 +674,7 @@ impl Validate for PartialManifest { Ok((trgt, path)) }) .collect::>>()?, + reexport_include_dirs: reexp_inc_dirs, plugins, frozen, workspace, @@ -895,6 +931,87 @@ impl Validate for PartialIncludeDir { } } +/// A partial entry of `export_include_dirs`. +/// +/// Either a directory of this package (`dir`), or the include directories of a +/// dependency (`dep`, optionally restricted to a single `dir` of that dependency). +#[derive(Serialize, Deserialize, Debug)] +pub struct PartialExportIncludeDir { + /// The target spec for which the include dir applies + pub target: Option, + /// The include directory path + pub dir: Option, + /// The dependency whose include directories are re-exported + pub dep: Option, + /// Unknown extra fields + #[serde(flatten)] + extra: HashMap, +} + +impl FromStr for PartialExportIncludeDir { + type Err = Void; + fn from_str(s: &str) -> std::result::Result { + Ok(PartialExportIncludeDir { + target: None, + dir: Some(s.into()), + dep: None, + extra: HashMap::new(), + }) + } +} + +impl PrefixPaths for PartialExportIncludeDir { + fn prefix_paths(self, prefix: &Path) -> Result { + // The `dir` of a re-export refers to the dependency, not to this package. + if self.dep.is_some() { + return Ok(self); + } + Ok(PartialExportIncludeDir { + dir: self.dir.prefix_paths(prefix)?, + ..self + }) + } +} + +/// An entry of `export_include_dirs`. +pub enum ExportIncludeDir { + /// An include directory of this package. + Local(TargetSpec, PathBuf), + /// Include directories of a dependency. + Dep(ReexportIncludeDir), +} + +impl Validate for PartialExportIncludeDir { + type Output = ExportIncludeDir; + type Error = Error; + fn validate(self, vctx: &ValidationContext) -> Result { + if !vctx.pre_output { + self.extra.iter().for_each(|(k, _)| { + Warnings::IgnoreUnknownField { + field: k.clone(), + pkg: vctx.package_name.to_string(), + } + .emit(); + }); + } + let target = self.target.unwrap_or(TargetSpec::Wildcard); + match (self.dep, self.dir) { + (None, Some(dir)) => Ok(ExportIncludeDir::Local(target, env_path_from_string(&dir)?)), + (Some(_), Some(dir)) if dir.trim().is_empty() => Err(err!( + "The `dir` of a re-exported dependency must not be empty." + )), + (Some(dep), dir) => Ok(ExportIncludeDir::Dep(ReexportIncludeDir { + dep: dep.to_lowercase(), + dir: dir.map(PathBuf::from), + target, + })), + (None, None) => Err(err!( + "An export_include_dirs entry needs a `dir` or a `dep`." + )), + } + } +} + /// A partial filtered preprocessor define #[derive(Serialize, Deserialize, Debug, Default)] pub struct PartialDefine { diff --git a/src/diagnostic.rs b/src/diagnostic.rs index 4e11915f..272f7a3c 100644 --- a/src/diagnostic.rs +++ b/src/diagnostic.rs @@ -446,6 +446,18 @@ pub enum Warnings { ) )] SubmodulesDisabled(String, Vec), + + #[error("Package {} re-exports include directories of {}, which is not one of its dependencies.", fmt_pkg!(pkg), fmt_pkg!(dep))] + #[diagnostic(code(W37))] + ExportDirUnknownDep { pkg: String, dep: String }, + + #[error("Package {} re-exports include directory {} of {}, which does not match any of its `export_include_dirs`.", fmt_pkg!(pkg), fmt_path!(dir.display()), fmt_pkg!(dep))] + #[diagnostic(code(W38))] + ExportDirNotFound { + pkg: String, + dep: String, + dir: PathBuf, + }, } #[derive(Error, Diagnostic, Debug, Clone)] @@ -469,6 +481,24 @@ pub enum Errors { #[error("Path {} for dependency {} does not exist.", fmt_path!(path.display()), fmt_pkg!(pkg))] #[diagnostic(code(E32))] DepPathMissing { pkg: String, path: PathBuf }, + + #[error("Package {} re-exports include directories of {}, which is not one of its dependencies.", fmt_pkg!(pkg), fmt_pkg!(dep))] + #[diagnostic( + code(E33), + help("Add {} to the `dependencies` of {}, or remove the entry from `export_include_dirs`.", fmt_pkg!(dep), fmt_pkg!(pkg)) + )] + ExportDirUnknownDep { pkg: String, dep: String }, + + #[error("Package {} re-exports include directory {} of {}, which does not match any of its `export_include_dirs`.", fmt_pkg!(pkg), fmt_path!(dir.display()), fmt_pkg!(dep))] + #[diagnostic( + code(E34), + help("The `dir` must be listed in the `export_include_dirs` of {} (relative to its root).", fmt_pkg!(dep)) + )] + ExportDirNotFound { + pkg: String, + dep: String, + dir: PathBuf, + }, } impl Errors { @@ -478,6 +508,10 @@ impl Errors { Errors::IgnoredPath { cause } => Warnings::IgnoredPath { cause }, Errors::FileMissing { path } => Warnings::FileMissing { path }, Errors::DepPathMissing { pkg, path } => Warnings::DepPathMissing { pkg, path }, + Errors::ExportDirUnknownDep { pkg, dep } => Warnings::ExportDirUnknownDep { pkg, dep }, + Errors::ExportDirNotFound { pkg, dep, dir } => { + Warnings::ExportDirNotFound { pkg, dep, dir } + } } } diff --git a/src/sess.rs b/src/sess.rs index 272933d0..33891ac0 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1723,31 +1723,78 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { use std::iter::once; - // Create IndexMap of the export_include_dirs for each package + // Create IndexMap of the export_include_dirs for each package, including + // the directories re-exported from dependencies. Ranks are ordered such + // that dependencies come before their dependents, so the exports of a + // dependency are final by the time they are re-exported. let mut all_export_include_dirs: IndexMap> = IndexMap::new(); - let tmp_export_include_dirs: Vec> = ranks - .clone() - .into_iter() - .chain(once(vec![Some(self.sess.manifest)])) - .map(|manifests| { + let pkgs = self.sess.packages(); + let manifests_with_roots = ranks + .iter() + .zip(pkgs.iter()) + .flat_map(|(manifests, deps)| { manifests - .into_iter() - .flatten() - .map(|m| { + .iter() + .zip(deps.iter()) + .filter_map(|(m, &dep_id)| m.map(|m| (m, self.sess.get_package_path(dep_id)))) + }) + .chain(once((self.sess.manifest, self.sess.root.to_path_buf()))) + .collect::>(); + for (m, _) in &manifests_with_roots { + let mut dirs: Vec<(TargetSpec, &Path)> = m + .export_include_dirs + .iter() + .map(|(trgt, path)| (trgt.clone(), path.as_path())) + .collect(); + for re in &m.reexport_include_dirs { + let Some(dep_dirs) = all_export_include_dirs.get(&re.dep) else { + Warnings::ExportDirNameIssue(re.dep.clone()).emit(); + continue; + }; + // `dir` must exactly match an export dir of the dependency, + // relative to the dependency root. + let wanted = re.dir.as_deref().map(normalize_rel_path); + let dep_root = manifests_with_roots + .iter() + .find(|(dm, _)| dm.package.name == re.dep) + .map(|(_, root)| root.as_path()); + let selected: Vec<_> = dep_dirs + .iter() + .filter(|(_, path)| match &wanted { + None => true, + Some(wanted) => { + wanted.is_some() + && dep_root + .and_then(|root| path.strip_prefix(root).ok()) + .and_then(normalize_rel_path) + == *wanted + } + }) + .map(|(trgt, path)| { ( - m.package.name.clone(), - m.export_include_dirs - .iter() - .map(|(trgt, path)| (trgt.clone(), path.as_path())) - .collect(), + TargetSpec::All([re.target.clone(), trgt.clone()].into()).reduce(), + *path, ) }) - .collect() - }) - .collect(); - for element in tmp_export_include_dirs { - all_export_include_dirs.extend(element); + .collect(); + match &re.dir { + Some(dir) if selected.is_empty() => Errors::ExportDirNotFound { + pkg: m.package.name.clone(), + dep: re.dep.clone(), + dir: dir.clone(), + } + .downgrade_if_suppressed()?, + _ => { + for d in selected { + if !dirs.contains(&d) { + dirs.push(d); + } + } + } + } + } + all_export_include_dirs.insert(m.package.name.clone(), dirs); } log::debug!( "export_include_dirs for each package: {:?}", @@ -1778,8 +1825,16 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { Warnings::ExportDirNameIssue(i.clone()).emit(); export_include_dirs.insert(i.to_string(), Vec::new()); } else { - export_include_dirs - .insert(i.to_string(), all_export_include_dirs[i].clone()); + // Skip directories already listed for another package, + // e.g. when several dependencies re-export the same one. + let dirs = all_export_include_dirs[i] + .iter() + .filter(|d| { + !export_include_dirs.values().flatten().any(|e| e == *d) + }) + .cloned() + .collect(); + export_include_dirs.insert(i.to_string(), dirs); } } } @@ -2263,3 +2318,16 @@ pub struct Plugin { /// What binary implements the plugin. pub path: PathBuf, } + +/// Normalize a relative path by dropping `.` components and trailing slashes. +/// +/// Returns `None` for paths that are absolute or contain `..`. +fn normalize_rel_path(path: &Path) -> Option { + path.components() + .filter(|c| !matches!(c, std::path::Component::CurDir)) + .map(|c| match c { + std::path::Component::Normal(n) => Some(n), + _ => None, + }) + .collect() +} diff --git a/tests/transitive_export_include_dirs_test.sh b/tests/transitive_export_include_dirs_test.sh new file mode 100755 index 00000000..44ebdbf2 --- /dev/null +++ b/tests/transitive_export_include_dirs_test.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Test re-exporting include directories of dependencies via `export_include_dirs`. +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")" +BENDER=${BENDER:-"cargo run --"} +DIR="$(pwd)"/tmp/"$(basename $0 _test.sh)" +[ ! -d "$DIR" ] || rm -rf "$DIR" +mkdir -p "$DIR" +cd "$DIR" + +fail() { + echo "$1" >&2 + exit 1 +} + +# c exports `inc` and `other`; b re-exports `inc` of c; a re-exports all of b. +mkdir -p c/inc c/other c/sub/inc c/src b/inc b/src a/src top/src +touch c/src/c.sv b/src/b.sv a/src/a.sv top/src/top.sv +cat >c/Bender.yml <b/Bender.yml <a/Bender.yml <top/Bender.yml <log; then fail "dir $d should fail"; fi + grep -q "does not match any of its" log || { cat log; fail "wrong error for dir $d"; } + cp ../b/Bender.yml.orig ../b/Bender.yml +done + +# An empty `dir` is rejected. +sed -i.bak 's|dir: inc|dir: ""|' ../b/Bender.yml +if $BENDER script vsim &>log; then fail "empty dir should fail"; fi +cp ../b/Bender.yml.orig ../b/Bender.yml + +# Target filtering of a re-export is honored. +sed -i.bak 's| - dep: c| - dep: c\n target: sim|' ../b/Bender.yml +incdirs_of src/top.sv | grep -q "c/inc" && fail "c/inc must be filtered without target sim" +incdirs_of src/top.sv -t sim | grep -q "+incdir+.*/c/inc\"" || fail "missing c/inc with target sim" +cp ../b/Bender.yml.orig ../b/Bender.yml + +# A directory the dependency does not export is an error. +sed -i.bak 's|dir: inc|dir: nope|' ../b/Bender.yml +if $BENDER script vsim &>log; then fail "unknown dir should fail"; fi +grep -q "does not match any of its" log || { cat log; fail "wrong error for unknown dir"; } +cp ../b/Bender.yml.orig ../b/Bender.yml + +# A package that is not a dependency is an error. +sed -i.bak 's|dep: c|dep: z|' ../b/Bender.yml +if $BENDER script vsim &>log; then fail "unknown dep should fail"; fi +grep -q "not one of its dependencies" log || { cat log; fail "wrong error for unknown dep"; } + +# Diamond: top depends on b and e, which both re-export c; no duplicates. +cp ../b/Bender.yml.orig ../b/Bender.yml +mkdir -p ../e/src +touch ../e/src/e.sv +cat >../e/Bender.yml <Bender.yml </dev/null +n=$(incdirs_of src/top.sv | grep -c "/c/inc\"") +[ "$n" = 1 ] || fail "c/inc listed $n times in diamond"