extract_identifiers_clang_json: factor out Clang AST logic. The goal is to create multiple main programs for the different use cases, so a rather simple one can be used to generate the public symbol list. Also, this makes it a valid option to possibly embed it into the pregenerate tool. Also, export whether a function is inline, as inline functions can't be redefine_extname'd. Bug: 42220000 Change-Id: I8aaedd10f03d688bd808b3125340cd1d6a6a6964 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/86947 Reviewed-by: Xiangfei Ding <xfding@google.com> Commit-Queue: Rudolf Polzer <rpolzer@google.com>
diff --git a/util/extract_identifiers_clang_json.go b/util/extract_identifiers_clang_json.go index 594b7d5..e63d9a2 100644 --- a/util/extract_identifiers_clang_json.go +++ b/util/extract_identifiers_clang_json.go
@@ -28,14 +28,12 @@ package main import ( - "encoding/json" "flag" "fmt" "log" "os" - "path" - "regexp" - "strings" + + "boringssl.googlesource.com/boringssl.git/util/idextractor" ) var ( @@ -46,578 +44,50 @@ globalSymbolsOnly = flag.Bool("global_symbols_only", false, "only output a list of names that may become (part of) linker symbols and are not namespaced") ) -// node is a node from the Clang AST dump. -type node struct { - Kind string - Loc loc - Range rangeStruct `json:",omitempty"` - Inner []*node `json:",omitempty"` - - // Node fields that may or may not matter depending on `Kind`. - CompleteDefinition bool `json:",omitempty"` - ConstExpr bool `json:",omitempty"` - Decl *node `json:",omitempty"` - IsImplicit bool `json:",omitempty"` - Language string `json:",omitempty"` - Name string `json:",omitempty"` - PreviousDecl string `json:",omitempty"` - StorageClass string `json:",omitempty"` - TagUsed string `json:",omitempty"` - Type *typeStruct `json:",omitempty"` +func reportIdentifierGlobalSymbolsOnly(id idextractor.IdentifierInfo) error { + // NOTE: This is over-exporting. + // At least for quite a while we can't namespace the + // enum, struct and union tags either + // as they may be forward declared by callers. + // The idea is to first get this overzealous namespacing to work, + // and then to remove those forward declaration prone symbols + // at least initially (and possibly namespace them then one by one). + if id.Symbol == id.Identifier && id.Linkage != "static" && id.Tag != "typedef" && id.Tag != "using" { + fmt.Printf("%s\n", id.Identifier) + } + return nil } -// typeStruct is the type information from the Clang AST dump. -type typeStruct struct { - QualType string `json:",omitempty"` - DesugaredQualType string `json:",omitempty"` -} +var seen = map[string]string{} -// desugaredQualType returns the canonical type after expanding typedefs. -func (t typeStruct) desugaredQualType() string { - if t.DesugaredQualType != "" { - return t.DesugaredQualType +func reportIdentifierVerbose(id idextractor.IdentifierInfo) error { + linkage := "" + if id.Linkage != "" { + linkage = id.Linkage + " " } - return t.QualType -} - -var typeTag = regexp.MustCompile(`^(?:enum|struct|union)\b`) - -// qualTypeTag returns the tag of the qualified type, if any. -func (t typeStruct) qualTypeTag() string { - return typeTag.FindString(t.QualType) -} - -// constTypeRE is an approximate regex for types that are const. -// It errs on the side of returning constants as non-const. -var constType = regexp.MustCompile(`.*\bconst\b[^\[\]()*&]*$`) - -// isConst returns whether the given type is likely a constant. -func (t typeStruct) isConst() bool { - return constType.MatchString(t.desugaredQualType()) -} - -// rangeStruct is a location range from the Clang AST dump. -type rangeStruct struct { - Begin *loc `json:",omitempty"` - End *loc `json:",omitempty"` -} - -// loc is a location from the Clang AST dump. -type loc struct { - File string `json:",omitempty"` - Line uint `json:",omitempty"` - Col uint `json:",omitempty"` - Offset uint `json:",omitempty"` - TokLen uint `json:",omitempty"` - SpellingLoc *loc `json:",omitempty"` - ExpansionLoc *loc `json:",omitempty"` -} - -// expansionLoc returns the expansion location of the given loc. -func (l loc) expansionLoc() loc { - if l.ExpansionLoc != nil { - return *l.ExpansionLoc - } - if l.SpellingLoc != nil { - return *l.SpellingLoc - } - return l -} - -type decompressCtx struct { - file string - line uint -} - -// decompress undoes the filename field compression from -// JSONNodeDumper::writeSourceLocation and JSONNodeDumper::writeBareSourceLocation. -func (l *loc) decompress(last *decompressCtx) { - if l == nil { - return - } - l.SpellingLoc.decompress(last) - l.ExpansionLoc.decompress(last) - if l.SpellingLoc != nil || l.ExpansionLoc != nil { - return - } - if l.File == "" { - l.File = last.file - } else { - last.file = l.File - } - if l.Line == 0 { - l.Line = last.line - } else { - last.line = l.Line - } -} - -// path returns the loc's file path in clean, unique form. -func (l loc) path() string { - return path.Clean(l.File) -} - -// decompressLocsInternal is a helper for decompressLocs. -// -// It keeps state in its lastFile pointer. -func (n *node) decompressLocsInternal(last *decompressCtx) { - n.Loc.decompress(last) - n.Range.Begin.decompress(last) - n.Range.End.decompress(last) - for _, child := range n.Inner { - child.decompressLocsInternal(last) - } -} - -// decompressLocs decompresses all Loc fields below a node. -// -// Should be called right after parsing. -func (n *node) decompressLocs() { - n.decompressLocsInternal(&decompressCtx{}) -} - -// storage represents the storage class of a node. -type storage int - -const ( - noStorage storage = iota - externStorage - staticStorage -) - -// storage finds the storage class of the node. -func (n node) storage(language string) (storage, error) { - var storage storage - switch n.StorageClass { - case "": - if n.Kind == "VarDecl" && (n.ConstExpr || (language == "C++" && n.Type.isConst())) { - storage = staticStorage - } else { - storage = externStorage - } - case "extern": - storage = externStorage - case "static": - storage = staticStorage - default: - return noStorage, fmt.Errorf("no handling for storage class %q", n.StorageClass) - } - return storage, nil -} - -// functionArgs returns all the function arg nodes of a node. -func (n node) functionArgs() []*node { - var result []*node - for _, child := range n.Inner { - if child.Kind == "ParmVarDecl" { - result = append(result, child) - } - } - return result -} - -func (n node) locationRange() (file string, line, start, end uint, ok bool) { - from := n.Range.Begin.expansionLoc() - to := n.Range.End.expansionLoc() - if from.path() != to.path() { - return "", 0, 0, 0, false - } - return from.path(), from.Line, from.Offset, to.Offset + to.TokLen, true -} - -func (n node) locationRangeWithoutBody() (file string, line, start, end uint, ok bool) { - file, line, start, end, ok = n.locationRange() - if !ok { - return - } - // Cut bad nodes from the end. - for i := len(n.Inner) - 1; i >= 0; i-- { - child := n.Inner[i] - if child.Kind != "CompoundStmt" { - continue - } - cfile, _, cstart, cend, ok := child.locationRange() - if !ok { - continue - } - if cfile == file && cend == end { - end = cstart - } - } - return -} - -// namespacing indicates how the identifier respects namespaces. -type namespacing int - -const ( - alwaysGlobal namespacing = iota // Never in namespace (such as preprocessor macros). - globalIfC // Respects namespace unless in extern "C" (such as functions). - alwaysNamespaced // Always respects namespace (such as types). -) - -// linking indicates how the identifier responds to extern "C" or similar. -type linking int - -const ( - neverLinked linking = iota // Ignores linkage information (such as types). - respectsLinkage // Respects linkage information (such as functions). -) - -// walker is data that is transported to inner nodes while parsing. -type walker struct { - *walkerStatic // Data that can be mutated even by downstream nodes. - - inBoringSSL bool // Whether the code originates from BoringSSL. - depth int // Nesting depth (for -dump_tree output). - namespace []string // C++ namespace sequence the node is in. - anonNamespace bool // Whether the node is in a C++ anonymous namespace. - language string // Can be "C" or "C++". - record bool // Whether the current node is part of a record. -} - -// walkerStatic is data that is transported in reading direction while parsing. -type walkerStatic struct { - seen map[string]string // All identifiers seen so far. -} - -func newWalker() walker { - return walker{ - walkerStatic: &walkerStatic{ - seen: map[string]string{}, - }, - language: *language, - } -} - -// Consider files with a non-absolute path to be BoringSSL, -// whereas absolute paths usually indicate system header locations. -// -// Note that any non-word character in the first two characters is treated as -// indicating an absolute path to catch "<built-in>", "/foo/bar.h" and "C:\foo\bar.h". -var ( - boringSSLPath = regexp.MustCompile(`^\w\w`) -) - -// updateInBoringSSL checks whether the given directive is a file/line directive, -// and if so, checks if it's likely part of BoringSSL or not. -// -// The return value indicates whether it's a file/line directive. -// If it is, `*in` will be updated to the current status of whether this is BoringSSL. -func (w *walker) updateInBoringSSL(kind string, loc loc) { - if kind == "TranslationUnitDecl" { - w.inBoringSSL = true - return - } - w.inBoringSSL = boringSSLPath.MatchString(loc.expansionLoc().path()) -} - -// visit traverses a node in the AST and analyzes it for identifiers contained therein. -func (w walker) visit(n *node) (err error) { - nodeWithoutChildren := *n - nodeWithoutChildren.Inner = nil - nodeCode, err := json.Marshal(nodeWithoutChildren) - if err != nil { - return err - } - - if (*dumpTree && w.inBoringSSL) || *dumpFullTree { - log.Printf("%*s[%s] %s: %s (%d children)", - w.depth, "", - strings.Join(w.namespace, "::"), - n.Kind, - nodeCode, - len(n.Inner)) - } - - // Allow to ignore errors. - defer func() { - if *keepGoing && err != nil { - log.Printf("ERROR: %v", err) - err = nil - } - }() - - // Update "w". - w.depth++ - - // Update "in BoringSSL". - w.updateInBoringSSL(n.Kind, n.Loc) - - if !w.inBoringSSL || n.IsImplicit { - // If suppressed, below nodes are not interesting. - // Also, skip any non-BoringSSL code such as system headers. - return nil - } - - switch n.Kind { - // Nodes that need handling. - case "CXXRecordDecl", "RecordDecl": - if w.record && n.CompleteDefinition { - return nil - } - if n.Name != "" { - if err := w.collectIdentifier(n, n.TagUsed, alwaysNamespaced, neverLinked, noStorage); err != nil { - return err - } - } - w.record = true - case "EnumDecl": - if w.record { - return nil - } - if n.Name != "" { - if err := w.collectIdentifier(n, "enum", alwaysNamespaced, neverLinked, noStorage); err != nil { - return err - } - } - case "EnumConstantDecl": - if w.record { - return nil - } - if err := w.collectIdentifier(n, "enumerator", alwaysNamespaced, neverLinked, noStorage); err != nil { - return err - } - return nil // Do not recurse. - case "FunctionDecl": - if w.record { - return nil - } - if n.PreviousDecl != "" { - return // Definition or redeclaration doesn't need to be looked at again (and may have incomplete qualifiers). - } - storage, err := n.storage(w.language) - if err != nil { - return fmt.Errorf("could not find storage class of function: %w: %s", err, nodeCode) - } - if err := w.collectIdentifier(n, "function", globalIfC, respectsLinkage, storage); err != nil { - return err - } - case "LinkageSpecDecl": - if n.Language != "" { - w.language = n.Language - } - case "NamespaceDecl": - if w.language != "C++" { - return fmt.Errorf("entering namespace while in extern %q is probably unintended: %s", w.language, nodeCode) - } - if n.Name == "" { - w.anonNamespace = true - } else { - w.namespace = append(append([]string(nil), w.namespace...), n.Name) - } - case "TypeAliasDecl", "TypeAliasTemplateDecl": - if w.record { - return nil - } - if err := w.collectIdentifier(n, "using", alwaysNamespaced, neverLinked, noStorage); err != nil { - return err - } - case "TypedefDecl": - if w.record { - return nil - } - if len(n.Inner) == 1 && n.Inner[0].Kind == "ElaboratedType" && len(n.Inner[0].Inner) == 1 && n.Inner[0].Inner[0].Decl != nil && n.Inner[0].Inner[0].Decl.Name == n.Name { - // typedef struct X { ... } X; - return nil - } - tag := "typedef" - if len(n.Inner) == 1 && n.Inner[0].Kind == "ElaboratedType" && len(n.Inner[0].Inner) == 1 && n.Inner[0].Inner[0].Decl != nil && n.Inner[0].Inner[0].Decl.Name == "" { - tag = n.Type.qualTypeTag() - if tag == "" { - return fmt.Errorf("typedef refers to an anonymous type but has no tag: %s", nodeCode) - } - } - if err := w.collectIdentifier(n, tag, alwaysNamespaced, neverLinked, noStorage); err != nil { - return err - } - case "VarDecl": - if n.PreviousDecl != "" { - return // Definition or redeclaration doesn't need to be looked at again (and may have incomplete qualifiers). - } - storage, err := n.storage(w.language) - if err != nil { - return fmt.Errorf("could not find storage class of variable: %w: %s", err, nodeCode) - } - if err := w.collectIdentifier(n, "var", globalIfC, respectsLinkage, storage); err != nil { - return err - } - return nil // Do not recurse. (Maybe should, to catch `struct ...` in variable types?) - // Singletons that should be skipped. - case - "AccessSpecDecl", - "AlwaysInlineAttr", - "BuiltinAttr", - "BuiltinType", - "CXX11NoReturnAttr", - "ConstAttr", - "DependentNameType", - "DeprecatedAttr", - "EnumType", - "FinalAttr", - "FormatAttr", - "NoThrowAttr", - "RecordType", - "TemplateTypeParmType", - "UnresolvedUsingValueDecl", - "UnusedAttr", - "UsingDecl", - "UsingDirectiveDecl", - "VisibilityAttr", - "WarnUnusedResultAttr", - "WeakAttr": - if len(n.Inner) != 0 { - // If this ever fires, check AST to see if any of the node's children could be useful, - // then categorize the node type into one of the following two cases. - return fmt.Errorf("singleton node of kind %q has children: %s", n.Kind, nodeCode) - } - // Nodes that should be skipped including possible children. - case - "AlignedAttr", - "CXXConstructorDecl", - "CXXConversionDecl", - "CXXDeductionGuideDecl", - "CXXDestructorDecl", - "CXXMethodDecl", - "ClassTemplatePartialSpecializationDecl", - "ClassTemplateSpecializationDecl", - "CompoundStmt", - "DecltypeType", - "FieldDecl", - "FriendDecl", - "NonTypeTemplateParmDecl", - "ParmVarDecl", - "StaticAssertDecl", - "TemplateArgument", - "TemplateTemplateParmDecl", - "TemplateTypeParmDecl", - "VarTemplateDecl", - "VarTemplateSpecializationDecl": - return nil // Do not recurse. - // Nodes that should just be recursed into. - case - "ClassTemplateDecl", - "ConstantArrayType", - "DecayedType", - "ElaboratedType", - "FunctionProtoType", - "FunctionTemplateDecl", - "IncompleteArrayType", - "IndirectFieldDecl", - "LValueReferenceType", - "ParenType", - "PointerType", - "QualType", - "TemplateSpecializationType", - "TranslationUnitDecl", - "TypedefType", - "VectorType": - // Just recurse. - default: - return fmt.Errorf("no handling for node kind %q: %s", n.Kind, nodeCode) - } - - // If we get here (via fallthrough usually), we want to recurse. - // To avoid recursing, use return. - for _, child := range n.Inner { - err = w.visit(child) - if err != nil { - break - } - } - - return err -} - -// collectIdentifier sends an identifier to the output. -func (w walker) collectIdentifier(n *node, tag string, namespacing namespacing, linking linking, storage storage) error { - name := n.Name - var fqn string - if w.anonNamespace { - fqn = "<anonymous>::" + name - } else { - fqn = strings.Join(append(append([]string(nil), w.namespace...), name), "::") - } - - // With this taken out of the way, for all intents and purposes - // anything in an anonymous namespace behaves as if it were static. - // - // Helps in some cases where the static keyword - // isn't repeated in template specializations or similar. - if w.anonNamespace && w.language == "C++" { - storage = staticStorage - } - - var linkage string - switch linking { - case neverLinked: - linkage = "" - case respectsLinkage: - switch storage { - case externStorage: - linkage = fmt.Sprintf("extern %q ", w.language) - case staticStorage: - linkage = "static " - default: - return fmt.Errorf("respecting linkage, but storage not set for %v", fqn) - } - } - - var namespaced bool - switch namespacing { - case alwaysGlobal: - namespaced = false - case globalIfC: - namespaced = w.language == "C++" - case alwaysNamespaced: - namespaced = true - } - - var identifier string - if namespaced { - identifier = fqn - } else { - identifier = name - } - - declaration := fmt.Sprintf("%s%s %s;", linkage, tag, identifier) - key := identifier - seen, found := w.seen[key] + declaration := fmt.Sprintf("%s%s %s;", linkage, id.Tag, id.Symbol) + key := id.Symbol + previous, found := seen[key] if found { - if seen != declaration { - return fmt.Errorf("duplicate distinct definition of %v: %v and %v", key, seen, declaration) + if previous != declaration { + return fmt.Errorf("duplicate distinct definition of %v: %v and %v", key, previous, declaration) } return nil } - w.seen[key] = declaration - - if *globalSymbolsOnly { - // NOTE: This is over-exporting. - // At least for quite a while we can't namespace the - // enum, struct and union tags either - // as they may be forward declared by callers. - // The idea is to first get this overzealous namespacing to work, - // and then to remove those forward declaration prone symbols - // at least initially (and possibly namespace them then one by one). - if (!namespaced || len(w.namespace) == 0) && storage != staticStorage && tag != "typedef" && tag != "using" { - fmt.Printf("%s\n", identifier) - } - return nil - } + seen[key] = declaration // Append some debug info. // This might be used later to generate symbol renaming headers, // but is generally useful to humans debugging this tool's output. var suffix string - if file, line, start, end, ok := n.locationRangeWithoutBody(); ok { - suffix += fmt.Sprintf(" %s:%v (%v-%v)", file, line, start, end) + if id.File != "" { + suffix += fmt.Sprintf(" %s:%v (%v-%v)", id.File, id.Line, id.Start, id.End) } - for _, arg := range n.functionArgs() { - argName := arg.Name - if argName == "" { - argName = "_" + for _, arg := range id.FunctionArgs { + if arg == "" { + arg = "_" } - suffix += " " + argName + suffix += " " + arg } if suffix != "" { suffix = " //" + suffix @@ -628,24 +98,17 @@ // Main is the main program. func Main() error { - j := json.NewDecoder(os.Stdin) - - w := newWalker() - - for j.More() { - var root node - err := j.Decode(&root) - if err != nil { - return err - } - root.decompressLocs() - err = w.visit(&root) - if err != nil { - return err - } + report := reportIdentifierVerbose + if *globalSymbolsOnly { + report = reportIdentifierGlobalSymbolsOnly } - - return nil + x := idextractor.New(report, idextractor.Options{ + DumpTree: *dumpTree, + DumpFullTree: *dumpFullTree, + KeepGoing: *keepGoing, + Language: *language, + }) + return x.Parse(os.Stdin) } // main runs Main turning errors into exit codes.
diff --git a/util/idextractor/clang_ast_parser.go b/util/idextractor/clang_ast_parser.go new file mode 100644 index 0000000..6d6aa2e --- /dev/null +++ b/util/idextractor/clang_ast_parser.go
@@ -0,0 +1,563 @@ +// Copyright (c) 2025 The BoringSSL Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implementation to extract identifier declarations from a Clang AST. + +package idextractor + +import ( + "encoding/json" + "fmt" + "log" + "path" + "regexp" + "strings" +) + +// desugaredQualType returns the canonical type after expanding typedefs. +func (t typeStruct) desugaredQualType() string { + if t.DesugaredQualType != "" { + return t.DesugaredQualType + } + return t.QualType +} + +var typeTag = regexp.MustCompile(`^(?:enum|struct|union)\b`) + +// qualTypeTag returns the tag of the qualified type, if any. +func (t typeStruct) qualTypeTag() string { + return typeTag.FindString(t.QualType) +} + +// constTypeRE is an approximate regex for types that are const. +// It errs on the side of returning constants as non-const. +var constType = regexp.MustCompile(`.*\bconst\b[^\[\]()*&]*$`) + +// isConst returns whether the given type is likely a constant. +func (t typeStruct) isConst() bool { + return constType.MatchString(t.desugaredQualType()) +} + +// expansionLoc returns the expansion location of the given loc. +func (l loc) expansionLoc() loc { + if l.ExpansionLoc != nil { + return *l.ExpansionLoc + } + if l.SpellingLoc != nil { + return *l.SpellingLoc + } + return l +} + +type decompressCtx struct { + file string + line uint +} + +// decompress undoes the filename field compression from +// JSONNodeDumper::writeSourceLocation and JSONNodeDumper::writeBareSourceLocation. +func (l *loc) decompress(last *decompressCtx) { + if l == nil { + return + } + l.SpellingLoc.decompress(last) + l.ExpansionLoc.decompress(last) + if l.SpellingLoc != nil || l.ExpansionLoc != nil { + return + } + if l.File == "" { + l.File = last.file + } else { + last.file = l.File + } + if l.Line == 0 { + l.Line = last.line + } else { + last.line = l.Line + } +} + +// path returns the loc's file path in clean, unique form. +func (l loc) path() string { + return path.Clean(l.File) +} + +// decompressLocsInternal is a helper for decompressLocs. +// +// It keeps state in its lastFile pointer. +func (n *node) decompressLocsInternal(last *decompressCtx) { + n.Loc.decompress(last) + n.Range.Begin.decompress(last) + n.Range.End.decompress(last) + for _, child := range n.Inner { + child.decompressLocsInternal(last) + } +} + +// decompressLocs decompresses all Loc fields below a node. +// +// Should be called right after parsing. +func (n *node) decompressLocs() { + n.decompressLocsInternal(&decompressCtx{}) +} + +// storage represents the storage class of a node. +type storage int + +const ( + noStorage storage = iota + externStorage + staticStorage +) + +// storage finds the storage class of the node. +func (n node) storage(language string) (storage, error) { + var storage storage + switch n.StorageClass { + case "": + if n.Kind == "VarDecl" && (n.ConstExpr || (language == "C++" && n.Type.isConst())) { + storage = staticStorage + } else { + storage = externStorage + } + case "extern": + storage = externStorage + case "static": + storage = staticStorage + default: + return noStorage, fmt.Errorf("no handling for storage class %q", n.StorageClass) + } + return storage, nil +} + +// functionArgs returns all the function arg nodes of a node. +func (n node) functionArgs() []*node { + var result []*node + for _, child := range n.Inner { + if child.Kind == "ParmVarDecl" { + result = append(result, child) + } + } + return result +} + +func (n node) locationRange() (file string, line, start, end uint, ok bool) { + if n.Range.Begin == nil || n.Range.End == nil { + return "", 0, 0, 0, false + } + from := n.Range.Begin.expansionLoc() + to := n.Range.End.expansionLoc() + if from.path() != to.path() { + return "", 0, 0, 0, false + } + return from.path(), from.Line, from.Offset, to.Offset + to.TokLen, true +} + +func (n node) locationRangeWithoutBody() (file string, line, start, end uint, ok bool) { + file, line, start, end, ok = n.locationRange() + if !ok { + return + } + // Cut bad nodes from the end. + for i := len(n.Inner) - 1; i >= 0; i-- { + child := n.Inner[i] + if child.Kind != "CompoundStmt" { + continue + } + cfile, _, cstart, cend, ok := child.locationRange() + if !ok { + continue + } + if cfile == file && cend == end { + end = cstart + } + } + return +} + +// namespacing indicates how the identifier respects namespaces. +type namespacing int + +const ( + alwaysGlobal namespacing = iota // Never in namespace (such as preprocessor macros). + globalIfC // Respects namespace unless in extern "C" (such as functions). + alwaysNamespaced // Always respects namespace (such as types). +) + +// linking indicates how the identifier responds to extern "C" or similar. +type linking int + +const ( + neverLinked linking = iota // Ignores linkage information (such as types). + respectsLinkage // Respects linkage information (such as functions). +) + +// extractor is data that is transported to inner nodes while parsing. +type extractor struct { + *extractorStatic // Data that can be mutated even by downstream nodes. + + inBoringSSL bool // Whether the code originates from BoringSSL. + depth int // Nesting depth (for -dump_tree output). + namespace []string // C++ namespace sequence the node is in. + anonNamespace bool // Whether the node is in a C++ anonymous namespace. + language string // Can be "C" or "C++". + record bool // Whether the current node is part of a record. +} + +type IdentifierInfo struct { + Identifier string // Just the identifier. + Symbol string // Name as seen by linker. + FQName string // Fully qualified name in C++. + Linkage string // Linkage string (can be stuff like 'extern "C"' or "static"). + Tag string // Tag (`struct`, `union`, `var` etc.) + + File string // File where the identifier is declared. + Line uint // Line in the file. + Start uint // Start byte position in the file. + End uint // End byte position in the file. + FunctionArgs []string // List of function arguments by name. +} + +// extractorStatic is data that is transported in reading direction while parsing. +type extractorStatic struct { + reportIdentifier func(IdentifierInfo) error + options Options +} + +// Consider files with a non-absolute path to be BoringSSL, +// whereas absolute paths usually indicate system header locations. +// +// Note that any non-word character in the first two characters is treated as +// indicating an absolute path to catch "<built-in>", "/foo/bar.h" and "C:\foo\bar.h". +var ( + boringSSLPath = regexp.MustCompile(`^\w\w`) +) + +// updateInBoringSSL checks whether the given directive is a file/line directive, +// and if so, checks if it's likely part of BoringSSL or not. +// +// The return value indicates whether it's a file/line directive. +// If it is, `*in` will be updated to the current status of whether this is BoringSSL. +func (x *extractor) updateInBoringSSL(kind string, loc loc) { + if kind == "TranslationUnitDecl" { + x.inBoringSSL = true + return + } + x.inBoringSSL = boringSSLPath.MatchString(loc.expansionLoc().path()) +} + +// visit traverses a node in the AST and analyzes it for identifiers contained therein. +func (x extractor) visit(n *node) (err error) { + nodeWithoutChildren := *n + nodeWithoutChildren.Inner = nil + nodeCode, err := json.Marshal(nodeWithoutChildren) + if err != nil { + return err + } + + if (x.options.DumpTree && x.inBoringSSL) || x.options.DumpFullTree { + log.Printf("%*s[%s] %s: %s (%d children)", + x.depth, "", + strings.Join(x.namespace, "::"), + n.Kind, + nodeCode, + len(n.Inner)) + } + + // Allow to ignore errors. + defer func() { + if x.options.KeepGoing && err != nil { + log.Printf("ERROR: %v", err) + err = nil + } + }() + + // Update "x". + x.depth++ + + // Update "in BoringSSL". + x.updateInBoringSSL(n.Kind, n.Loc) + + if !x.inBoringSSL || n.IsImplicit { + // If suppressed, below nodes are not interesting. + // Also, skip any non-BoringSSL code such as system headers. + return nil + } + + switch n.Kind { + // Nodes that need handling. + case "CXXRecordDecl", "RecordDecl": + if x.record && n.CompleteDefinition { + return nil + } + if n.Name != "" { + if err := x.collectIdentifier(n, n.TagUsed, alwaysNamespaced, neverLinked, noStorage); err != nil { + return err + } + } + x.record = true + case "EnumDecl": + if x.record { + return nil + } + if n.Name != "" { + if err := x.collectIdentifier(n, "enum", alwaysNamespaced, neverLinked, noStorage); err != nil { + return err + } + } + case "EnumConstantDecl": + if x.record { + return nil + } + if err := x.collectIdentifier(n, "enumerator", alwaysNamespaced, neverLinked, noStorage); err != nil { + return err + } + return nil // Do not recurse. + case "FunctionDecl": + if x.record { + return nil + } + if n.PreviousDecl != "" { + return // Definition or redeclaration doesn't need to be looked at again (and may have incomplete qualifiers). + } + storage, err := n.storage(x.language) + if err != nil { + return fmt.Errorf("could not find storage class of function: %x: %s", err, nodeCode) + } + if err := x.collectIdentifier(n, "function", globalIfC, respectsLinkage, storage); err != nil { + return err + } + case "LinkageSpecDecl": + if n.Language != "" { + x.language = n.Language + } + case "NamespaceDecl": + if x.language != "C++" { + return fmt.Errorf("entering namespace while in extern %q is probably unintended: %s", x.language, nodeCode) + } + if n.Name == "" { + x.anonNamespace = true + } else { + x.namespace = append(append([]string(nil), x.namespace...), n.Name) + } + case "TypeAliasDecl", "TypeAliasTemplateDecl": + if x.record { + return nil + } + if err := x.collectIdentifier(n, "using", alwaysNamespaced, neverLinked, noStorage); err != nil { + return err + } + case "TypedefDecl": + if x.record { + return nil + } + if len(n.Inner) == 1 && n.Inner[0].Kind == "ElaboratedType" && len(n.Inner[0].Inner) == 1 && n.Inner[0].Inner[0].Decl != nil && n.Inner[0].Inner[0].Decl.Name == n.Name { + // typedef struct X { ... } X; + return nil + } + tag := "typedef" + if len(n.Inner) == 1 && n.Inner[0].Kind == "ElaboratedType" && len(n.Inner[0].Inner) == 1 && n.Inner[0].Inner[0].Decl != nil && n.Inner[0].Inner[0].Decl.Name == "" { + tag = n.Type.qualTypeTag() + if tag == "" { + return fmt.Errorf("typedef refers to an anonymous type but has no tag: %s", nodeCode) + } + } + if err := x.collectIdentifier(n, tag, alwaysNamespaced, neverLinked, noStorage); err != nil { + return err + } + case "VarDecl": + if n.PreviousDecl != "" { + return // Definition or redeclaration doesn't need to be looked at again (and may have incomplete qualifiers). + } + storage, err := n.storage(x.language) + if err != nil { + return fmt.Errorf("could not find storage class of variable: %x: %s", err, nodeCode) + } + if err := x.collectIdentifier(n, "var", globalIfC, respectsLinkage, storage); err != nil { + return err + } + return nil // Do not recurse. (Maybe should, to catch `struct ...` in variable types?) + case "": + // Currently this happens to the function body of `bssl::PushToStack` on Windows. + log.Printf("WARNING: ignoring kind-less AST node: %s", nodeCode) + return nil + // Singletons that should be skipped. + case + "AccessSpecDecl", + "AlwaysInlineAttr", + "BuiltinAttr", + "BuiltinType", + "CXX11NoReturnAttr", + "ConstAttr", + "DependentNameType", + "DeprecatedAttr", + "EnumType", + "FinalAttr", + "FormatAttr", + "MaxFieldAlignmentAttr", + "NoThrowAttr", + "RecordType", + "TemplateTypeParmType", + "UnresolvedUsingValueDecl", + "UnusedAttr", + "UsingDecl", + "UsingDirectiveDecl", + "VisibilityAttr", + "WarnUnusedResultAttr", + "WeakAttr": + if len(n.Inner) != 0 { + // If this ever fires, check AST to see if any of the node's children could be useful, + // then categorize the node type into one of the following two cases. + return fmt.Errorf("singleton node of kind %q has children: %s", n.Kind, nodeCode) + } + // Nodes that should be skipped including possible children. + case + "AlignedAttr", + "CXXConstructorDecl", + "CXXConversionDecl", + "CXXDeductionGuideDecl", + "CXXDestructorDecl", + "CXXMethodDecl", + "ClassTemplatePartialSpecializationDecl", + "ClassTemplateSpecializationDecl", + "CompoundStmt", + "DecltypeType", + "FieldDecl", + "FriendDecl", + "NonTypeTemplateParmDecl", + "ParmVarDecl", + "StaticAssertDecl", + "TemplateArgument", + "TemplateTemplateParmDecl", + "TemplateTypeParmDecl", + "VarTemplateDecl", + "VarTemplateSpecializationDecl": + return nil // Do not recurse. + // Nodes that should just be recursed into. + case + "ClassTemplateDecl", + "ConstantArrayType", + "DecayedType", + "ElaboratedType", + "FunctionProtoType", + "FunctionTemplateDecl", + "IncompleteArrayType", + "IndirectFieldDecl", + "LValueReferenceType", + "ParenType", + "PointerType", + "QualType", + "TemplateSpecializationType", + "TranslationUnitDecl", + "TypedefType", + "VectorType": + // Just recurse. + default: + return fmt.Errorf("no handling for node kind %q: %s", n.Kind, nodeCode) + } + + // If we get here (via fallthrough usually), we want to recurse. + // To avoid recursing, use return. + for _, child := range n.Inner { + err = x.visit(child) + if err != nil { + break + } + } + + return err +} + +// collectIdentifier sends an identifier to the output. +func (x extractor) collectIdentifier(n *node, tag string, namespacing namespacing, linking linking, storage storage) error { + identifier := n.Name + var fqn string + if x.anonNamespace { + fqn = "<anonymous>::" + identifier + } else { + fqn = strings.Join(append(append([]string(nil), x.namespace...), identifier), "::") + } + + // With this taken out of the way, for all intents and purposes + // anything in an anonymous namespace behaves as if it were static. + // + // Helps in some cases where the static keyword + // isn't repeated in template specializations or similar. + if x.anonNamespace && x.language == "C++" { + storage = staticStorage + } + + var linkage string + switch linking { + case neverLinked: + linkage = "" + case respectsLinkage: + switch storage { + case externStorage: + linkage = fmt.Sprintf("extern %q", x.language) + case staticStorage: + linkage = "static" + default: + return fmt.Errorf("respecting linkage, but storage not set for %v", fqn) + } + } + + if n.Inline { + if linkage != "" { + linkage += " " + } + linkage += "inline" + } + + var namespaced bool + switch namespacing { + case alwaysGlobal: + namespaced = false + case globalIfC: + namespaced = x.language == "C++" + case alwaysNamespaced: + namespaced = true + } + + var symbol string + if namespaced { + symbol = fqn + } else { + symbol = identifier + } + + file, line, start, end, ok := n.locationRangeWithoutBody() + if !ok { + file = "" + } + + var args []string + for _, arg := range n.functionArgs() { + args = append(args, arg.Name) + } + + id := IdentifierInfo{ + Identifier: identifier, + Symbol: symbol, + FQName: fqn, + Linkage: linkage, + Tag: tag, + File: file, + Line: line, + Start: start, + End: end, + FunctionArgs: args, + } + return x.reportIdentifier(id) +}
diff --git a/util/idextractor/clang_ast_types.go b/util/idextractor/clang_ast_types.go new file mode 100644 index 0000000..35403c0 --- /dev/null +++ b/util/idextractor/clang_ast_types.go
@@ -0,0 +1,62 @@ +// Copyright (c) 2025 The BoringSSL Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines just the types representing the Clang AST. +// Methods are in clang_ast_parser.go. + +package idextractor + +// node is a node from the Clang AST dump. +type node struct { + Kind string + Loc loc + Range rangeStruct `json:",omitempty"` + Inner []*node `json:",omitempty"` + + // Node fields that may or may not matter depending on `Kind`. + CompleteDefinition bool `json:",omitempty"` + ConstExpr bool `json:",omitempty"` + Decl *node `json:",omitempty"` + Inline bool `json:",omitempty"` + IsImplicit bool `json:",omitempty"` + Language string `json:",omitempty"` + Name string `json:",omitempty"` + PreviousDecl string `json:",omitempty"` + StorageClass string `json:",omitempty"` + TagUsed string `json:",omitempty"` + Type *typeStruct `json:",omitempty"` +} + +// typeStruct is the type information from the Clang AST dump. +type typeStruct struct { + QualType string `json:",omitempty"` + DesugaredQualType string `json:",omitempty"` +} + +// rangeStruct is a location range from the Clang AST dump. +type rangeStruct struct { + Begin *loc `json:",omitempty"` + End *loc `json:",omitempty"` +} + +// loc is a location from the Clang AST dump. +type loc struct { + File string `json:",omitempty"` + Line uint `json:",omitempty"` + Col uint `json:",omitempty"` + Offset uint `json:",omitempty"` + TokLen uint `json:",omitempty"` + SpellingLoc *loc `json:",omitempty"` + ExpansionLoc *loc `json:",omitempty"` +}
diff --git a/util/idextractor/idextractor.go b/util/idextractor/idextractor.go new file mode 100644 index 0000000..85435cb --- /dev/null +++ b/util/idextractor/idextractor.go
@@ -0,0 +1,66 @@ +// Copyright (c) 2026 The BoringSSL Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Public API for Clang based identifier extraction. + +package idextractor + +import ( + "encoding/json" + "io" +) + +// Options are the options for tree walking. +type Options struct { + // DumpTree prints the tree as it is parsed, but only for BoringSSL code. + DumpTree bool + // DumpFullTree prints the tree as it is parsed, even for system headers. + DumpFullTree bool + // KeepGoing does not bail out on parse errors. + KeepGoing bool + // Language is the langauge to parse the AST as. + Language string +} + +// New creates a new identiifer extractor. +func New(reporter func(IdentifierInfo) error, options Options) *extractor { + x := &extractor{ + extractorStatic: &extractorStatic{ + reportIdentifier: reporter, + options: options, + }, + language: options.Language, + } + return x +} + +// Parse parses the Clang AST from the given reader. +// It will invoke the reporter function passed to New +// for each declared identifier encountered. +func (x extractor) Parse(r io.Reader) error { + j := json.NewDecoder(r) + for j.More() { + var root node + err := j.Decode(&root) + if err != nil { + return err + } + root.decompressLocs() + err = x.visit(&root) + if err != nil { + return err + } + } + return nil +}
diff --git a/util/list_unintended_exported_symbols.sh b/util/list_unintended_exported_symbols.sh index ee2bcd8..e8a4a58 100755 --- a/util/list_unintended_exported_symbols.sh +++ b/util/list_unintended_exported_symbols.sh
@@ -88,8 +88,10 @@ } public_c_includes() { + # Note: using |hdrs| of _all_ modules. The assumption is that all + # |hdrs| are public. jq -r ' - [.bcm, .crypto, .decrepit] | + del(.pki) | .[] | .hdrs | .[]? @@ -139,7 +141,7 @@ # These symbols are often declared to support the STL and are benign, # as they have C++ linkage and very specific arguments and thus # non-conflicting mangled names. - grep -vE 'extern "C\+\+" function (begin|end);' || true + grep -vE 'extern "C\+\+" inline function (begin|end);' || true } filter_enum() { @@ -157,7 +159,7 @@ fix_c_cc_include_deltas() { # OPENSSL_INLINE behaves differently in C and C++. - sed -e 's,^static ,extern "C" ,g' |\ + sed -e 's,^static inline ,extern "C" inline ,g' |\ filter_bssl |\ filter_stl |\ filter_enum |\ @@ -195,6 +197,7 @@ echo >&2 'Comparing C includes across including language...' fix_c_cc_include_deltas < "${workdir}"/include.c.ids > "${workdir}"/include.c.common.ids fix_c_cc_include_deltas < "${workdir}"/include.c_as_cc.ids > "${workdir}"/include.c_as_cc.common.ids +diff -u "${workdir}"/include.c.common.ids "${workdir}"/include.c_as_cc.common.ids # Check that no source file defines any public symbols that are not in the # public headers, namespaced or otherwise OK'd.