Skip to content

Build the nested IR on the frontend. #1203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Build the nested IR on the frontend. #1203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/crubit_feature.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ flagset::flags! {
/// Always specialize generics in cpp_api_from_rust, instead of doing composable bridging
/// when possible.
AlwaysSpecializeGenericsInCppApiFromRust,

/// Generate bindings using the nested IR.
UseNestedIr,
}
}

Expand Down Expand Up @@ -84,6 +87,7 @@ impl CrubitFeature {
Self::AlwaysSpecializeGenericsInCppApiFromRust => {
"always_specialize_generics_in_cpp_api_from_rust"
}
Self::UseNestedIr => "use_nested_ir",
}
}

Expand Down Expand Up @@ -112,6 +116,7 @@ impl CrubitFeature {
Self::AlwaysSpecializeGenericsInCppApiFromRust => {
"//features:always_specialize_generics_in_cpp_api_from_rust"
}
Self::UseNestedIr => "//features:use_nested_ir",
}
}
}
Expand Down Expand Up @@ -143,6 +148,7 @@ pub fn named_features(name: &[u8]) -> Option<flagset::FlagSet<CrubitFeature>> {
b"always_specialize_generics_in_cpp_api_from_rust" => {
CrubitFeature::AlwaysSpecializeGenericsInCppApiFromRust.into()
}
b"use_nested_ir" => CrubitFeature::UseNestedIr.into(),
_ => return None,
// LINT.ThenChange(//depot/rs_bindings_from_cc/importer.cc, //depot/features/BUILD)
};
Expand Down Expand Up @@ -261,6 +267,7 @@ mod tests {
| CrubitFeature::CheckDefaultInitialized
| CrubitFeature::LeadingColonsForCppType
| CrubitFeature::TemplateInstantiation
| CrubitFeature::UseNestedIr
);
}

Expand Down Expand Up @@ -296,6 +303,7 @@ mod tests {
| CrubitFeature::CheckDefaultInitialized
| CrubitFeature::LeadingColonsForCppType
| CrubitFeature::TemplateInstantiation
| CrubitFeature::UseNestedIr
);
}

Expand All @@ -316,6 +324,7 @@ mod tests {
| CrubitFeature::CheckDefaultInitialized
| CrubitFeature::LeadingColonsForCppType
| CrubitFeature::TemplateInstantiation
| CrubitFeature::UseNestedIr
);
}

Expand All @@ -337,6 +346,7 @@ mod tests {
| CrubitFeature::CheckDefaultInitialized
| CrubitFeature::LeadingColonsForCppType
| CrubitFeature::TemplateInstantiation
| CrubitFeature::UseNestedIr
);
}
}
1 change: 1 addition & 0 deletions rs_bindings_from_cc/generate_bindings/lifetime_defaults_transform.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,5 +817,6 @@ pub fn lifetime_defaults_transform(db: &BindingsGenerator) -> Result<IR> {
crubit_features: ir.flat_ir().crubit_features.clone(),
reexported_namespaces: ir.flat_ir().reexported_namespaces.clone(),
unstable_rust_features: ir.flat_ir().unstable_rust_features.clone(),
top_level_items: ir.flat_ir().top_level_items.clone(),
}))
}
115 changes: 111 additions & 4 deletions rs_bindings_from_cc/ir.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

#include "absl/algorithm/container.h"
#include "absl/base/nullability.h"
#include "absl/container/flat_hash_map.h"
#include "absl/log/check.h"
#include "absl/status/status.h"
#include "absl/strings/cord.h"
Expand Down Expand Up @@ -1210,6 +1211,17 @@ llvm::json::Value Record::ToJson() const {
{"is_aggregate", is_aggregate},
{"is_canonical_alias", is_canonical_alias},
{"child_item_ids", std::move(json_item_ids)},
{"children",
[&] {
llvm::json::Array json_children;
json_children.reserve(children.size());
for (const auto& child : children) {
json_children.push_back(std::visit(
[](const auto& alternative) { return alternative.ToJson(); },
child->as_variant()));
}
return json_children;
}()},
{"enclosing_item_id", enclosing_item_id},
{"must_bind", must_bind},
{"overloads_operator_delete", overloads_operator_delete},
Expand Down Expand Up @@ -1277,6 +1289,10 @@ flat_proto::Record Record::ToFlatProto() const {
proto.mutable_child_item_ids()->Reserve(child_item_ids.size());
for (const auto& child : child_item_ids)
proto.add_child_item_ids(child.value());
proto.mutable_children()->Reserve(children.size());
for (const auto& child : children) {
*proto.add_children() = crubit::ToFlatProto(*child);
}
if (enclosing_item_id)
proto.set_enclosing_item_id(enclosing_item_id->value());
proto.set_must_bind(must_bind);
Expand Down Expand Up @@ -1639,6 +1655,17 @@ llvm::json::Value Namespace::ToJson() const {
{"unknown_attr", unknown_attr},
{"owning_target", owning_target},
{"child_item_ids", std::move(json_item_ids)},
{"children",
[&] {
llvm::json::Array json_children;
json_children.reserve(children.size());
for (const auto& child : children) {
json_children.push_back(std::visit(
[](const auto& alternative) { return alternative.ToJson(); },
child->as_variant()));
}
return json_children;
}()},
{"enclosing_item_id", enclosing_item_id},
{"is_inline", is_inline},
{"must_bind", must_bind},
Expand Down Expand Up @@ -1669,6 +1696,10 @@ flat_proto::Namespace Namespace::ToFlatProto() const {
proto.mutable_child_item_ids()->Reserve(child_item_ids.size());
for (const auto& child : child_item_ids)
proto.add_child_item_ids(child.value());
proto.mutable_children()->Reserve(children.size());
for (const auto& child : children) {
*proto.add_children() = crubit::ToFlatProto(*child);
}
if (enclosing_item_id)
proto.set_enclosing_item_id(enclosing_item_id->value());
proto.set_is_inline(is_inline);
Expand All @@ -1682,7 +1713,8 @@ llvm::json::Value IR::ToJson() const {
std::vector<llvm::json::Value> json_items;
json_items.reserve(items.size());
for (const auto& item : items) {
std::visit([&](auto&& item) { json_items.push_back(item.ToJson()); }, item);
std::visit([&](auto&& item) { json_items.push_back(item.ToJson()); },
item.as_variant());
}
CHECK_EQ(json_items.size(), items.size());

Expand All @@ -1708,11 +1740,24 @@ llvm::json::Value IR::ToJson() const {
features_json[target.value()] = std::move(feature_array);
}

llvm::json::Object top_level_items_json;
for (const auto& [target, items] : top_level_items) {
llvm::json::Array items_json;
items_json.reserve(items.size());
for (const auto& item : items) {
items_json.push_back(std::visit(
[](const auto& alternative) { return alternative.ToJson(); },
item->as_variant()));
}
top_level_items_json[target.value()] = std::move(items_json);
}

llvm::json::Object result{
{"public_headers", public_headers},
{"current_target", current_target},
{"items", std::move(json_items)},
{"top_level_item_ids", std::move(top_level_item_ids_json)},
{"top_level_items", std::move(top_level_items_json)},
{"crubit_features", std::move(features_json)},
{"reexported_namespaces", reexported_namespaces},
{"unstable_rust_features", unstable_rust_features},
Expand Down Expand Up @@ -1753,7 +1798,7 @@ flat_proto::Item ToFlatProto(const IR::Item& item) {
[&](const ExistingRustType& i) {
*proto.mutable_existing_rust_type() = i.ToFlatProto();
}},
item);
item.as_variant());
return proto;
}

Expand All @@ -1766,6 +1811,11 @@ void IR::ToFlatProto(flat_proto::IRProto* proto) const {
for (const auto& h : public_headers)
*proto->add_public_headers() = h.ToFlatProto();
proto->set_current_target(current_target.value());
bool use_nested_ir = false;
if (auto it = crubit_features.find(current_target);
it != crubit_features.end()) {
use_nested_ir = it->second.contains("use_nested_ir");
}
proto->mutable_items()->Reserve(items.size());
for (const auto& item : items)
*proto->add_items() = crubit::ToFlatProto(item);
Expand All @@ -1774,6 +1824,15 @@ void IR::ToFlatProto(flat_proto::IRProto* proto) const {
list.mutable_item_ids()->Reserve(item_ids.size());
for (const auto& id : item_ids) list.add_item_ids(id.value());
}
if (use_nested_ir) {
for (const auto& [target, items] : top_level_items) {
auto& list = (*proto->mutable_top_level_items())[target.value()];
list.mutable_items()->Reserve(items.size());
for (const auto& item : items) {
*list.add_items() = crubit::ToFlatProto(*item);
}
}
}
if (!crate_root_path.empty()) proto->set_crate_root_path(crate_root_path);
for (const auto& [target, features] : crubit_features) {
auto& set = (*proto->mutable_crubit_features())[target.value()];
Expand All @@ -1789,12 +1848,60 @@ void IR::ToFlatProto(flat_proto::IRProto* proto) const {

std::string ItemToString(const IR::Item& item) {
return std::visit(
[&](auto&& item) { return llvm::formatv("{0}", item.ToJson()); }, item);
[&](auto&& item) { return llvm::formatv("{0}", item.ToJson()); },
item.as_variant());
}

void SetMustBindItem(IR::Item& item) {
// All IR::Item variants have a `must_bind` field.
std::visit([](auto& item_variant) { item_variant.must_bind = true; }, item);
std::visit([](auto& item_variant) { item_variant.must_bind = true; },
item.as_variant());
}

void IR::BuildTree() {
top_level_items.clear();
absl::flat_hash_map<ItemId, std::shared_ptr<Item>> item_map;
item_map.reserve(items.size());

for (const auto& item : items) {
auto shared_item = std::make_shared<Item>(item);
ItemId id = std::visit([](const auto& val) { return val.id; },
shared_item->as_variant());
item_map[id] = std::move(shared_item);
}

for (auto& [id, shared_item] : item_map) {
std::visit(visitor{[&](Record& r) {
r.children.reserve(r.child_item_ids.size());
for (const auto& child_id : r.child_item_ids) {
if (auto it = item_map.find(child_id);
it != item_map.end()) {
r.children.push_back(it->second);
}
}
},
[&](Namespace& ns) {
ns.children.reserve(ns.child_item_ids.size());
for (const auto& child_id : ns.child_item_ids) {
if (auto it = item_map.find(child_id);
it != item_map.end()) {
ns.children.push_back(it->second);
}
}
},
[](auto& other) {}},
shared_item->as_variant());
}

for (const auto& [target, item_ids] : top_level_item_ids) {
auto& list = top_level_items[target];
list.reserve(item_ids.size());
for (const auto& id : item_ids) {
if (auto it = item_map.find(id); it != item_map.end()) {
list.push_back(it->second);
}
}
}
}

} // namespace crubit
29 changes: 26 additions & 3 deletions rs_bindings_from_cc/ir.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@

namespace crubit {

struct Item;

namespace internal {
inline constexpr int kJsonIndent = 2;
} // namespace internal
Expand Down Expand Up @@ -881,6 +883,9 @@ struct Record {

// Set if this is [[deprecated]]. If no message was given, will be "".
std::optional<std::string> deprecated;

// TODO(b/523265360): Remove child_item_ids.
std::vector<std::shared_ptr<Item>> children;
};

// A forward-declared record (e.g. `struct Foo;`)
Expand Down Expand Up @@ -1073,6 +1078,9 @@ struct Namespace {
// Set if this is [[deprecated]]. If no message was given, will be "".
std::optional<std::string> deprecated;
std::optional<std::string> doc_comment;

// TODO(b/523265360): Remove child_item_ids.
std::vector<std::shared_ptr<Item>> children;
};

inline std::ostream& operator<<(std::ostream& o, const Namespace& n) {
Expand Down Expand Up @@ -1124,6 +1132,19 @@ inline std::ostream& operator<<(std::ostream& o,
return o << std::string(llvm::formatv("{0:2}", existing_rust_type.ToJson()));
}

struct Item
: public std::variant<Func, Record, IncompleteRecord, Enum, Constant,
TypeAlias, GlobalVar, UnsupportedItem, Comment,
Namespace, UseMod, ExistingRustType> {
using Base = std::variant<Func, Record, IncompleteRecord, Enum, Constant,
TypeAlias, GlobalVar, UnsupportedItem, Comment,
Namespace, UseMod, ExistingRustType>;
using Base::Base;

const Base& as_variant() const { return *this; }
Base& as_variant() { return *this; }
};

// A complete intermediate representation of bindings for publicly accessible
// declarations of a single C++ library.
struct IR {
Expand Down Expand Up @@ -1161,11 +1182,13 @@ struct IR {

BazelLabel current_target;

using Item = std::variant<Func, Record, IncompleteRecord, Enum, Constant,
TypeAlias, GlobalVar, UnsupportedItem, Comment,
Namespace, UseMod, ExistingRustType>;
using Item = ::crubit::Item;
std::vector<Item> items;
absl::flat_hash_map<BazelLabel, std::vector<ItemId>> top_level_item_ids;
absl::flat_hash_map<BazelLabel, std::vector<std::shared_ptr<Item>>>
top_level_items;

void BuildTree();
// Empty string signals that the bindings should be generated in the crate
// root. This is the default state.
//
Expand Down
8 changes: 8 additions & 0 deletions rs_bindings_from_cc/ir.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ message Record {
string deprecated = 38;
bool is_explicit_class_template_instantiation_definition = 39;
bool has_private_or_deleted_operator_delete = 40;
repeated Item children = 41;
}

message IncompleteRecord {
Expand Down Expand Up @@ -491,6 +492,7 @@ message Namespace {
bool must_bind = 11;
string deprecated = 12;
string doc_comment = 13;
repeated Item children = 14;
}

message UseMod {
Expand Down Expand Up @@ -538,6 +540,9 @@ message IRProto {
message FeatureSet {
repeated string features = 1;
}
message ItemList {
repeated Item items = 1;
}

repeated HeaderName public_headers = 1;
string current_target = 2;
Expand All @@ -547,4 +552,7 @@ message IRProto {
map<string, FeatureSet> crubit_features = 6;
repeated string unstable_rust_features = 7;
repeated string reexported_namespaces = 8;
// Lexical tree structure of the IR. We would like for this to fully
// replace `top_level_item_ids` eventually. (See b/523265360)
map<string, ItemList> top_level_items = 9;
}
Loading