| // Copyright (c) 2024, Google Inc. |
| // |
| // Permission to use, copy, modify, and/or distribute this software for any |
| // purpose with or without fee is hereby granted, provided that the above |
| // copyright notice and this permission notice appear in all copies. |
| // |
| // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| |
| // prepare_bcr_module prepares for a BCR release. It outputs a JSON |
| // configuration file that may be used by BCR's add_module tool. |
| package main |
| |
| import ( |
| "archive/tar" |
| "bytes" |
| "compress/gzip" |
| "crypto/sha256" |
| "encoding/json" |
| "flag" |
| "fmt" |
| "io" |
| "net/http" |
| "os" |
| "path/filepath" |
| "runtime" |
| "strings" |
| ) |
| |
| var ( |
| outDir = flag.String("out-dir", "", "The directory to place the script output, or a temporary directory if unspecified.") |
| numWorkers = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers") |
| |
| moduleOverride = flag.String("module-override", "", "The path to a file that overrides the MODULE.bazel file in the archve.") |
| presubmitOverride = flag.String("presubmit-override", "", "The path to a file that overrides the presubmit.yml file in the archve.") |
| skipArchiveCheck = flag.Bool("skip-archive-check", false, "Skips checking the release tarball against the (potentially unstable) archive tarball.") |
| pipe = flag.Bool("pipe", false, "Prints output suitable for writing to a pipe instead of a terminal") |
| |
| githubOrg = flag.String("github-org", "google", "The organization where the GitHub repository lives") |
| githubRepo = flag.String("github-repo", "boringssl", "The name of the GitHub repository") |
| moduleName = flag.String("module-name", "boringssl", "The name of the BCR module") |
| compatibilityLevel = flag.String("compatibility-level", "2", "The compatibility_level setting for the BCR module") |
| ) |
| |
| // A bcrConfig is a configuration file for BCR's add_module tool. This is |
| // undocumented but can be seen in the Module Python class. (The JSON struct is |
| // simply the object's __dict__.) |
| type bcrConfig struct { |
| Name string `json:"name"` |
| Version string `json:"version"` |
| CompatibilityLevel string `json:"compatibility_level"` |
| ModuleDotBazel *string `json:"module_dot_bazel"` |
| URL *string `json:"url"` |
| StripPrefix *string `json:"strip_prefix"` |
| Deps []string `json:"deps"` |
| Patches []string `json:"patches"` |
| PatchStrip int `json:"patch_strip"` |
| BuildFile *string `json:"build_file"` |
| PresubmitYml *string `json:"presubmit_yml"` |
| BuildTargets []string `json:"build_targets"` |
| TestModulePath *string `json:"test_module_path"` |
| TestModuleBuildTargets []string `json:"test_module_build_targets"` |
| TestModuleTestTargets []string `json:"test_module_test_targets"` |
| } |
| |
| func ptr[T any](t T) *T { return &t } |
| |
| func archiveURL(tag string) string { |
| return fmt.Sprintf("https://github.com/%s/%s/archive/refs/tags/%s.tar.gz", *githubOrg, *githubRepo, tag) |
| } |
| |
| func releaseURL(tag string) string { |
| return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s.tar.gz", *githubOrg, *githubRepo, tag, *githubRepo, tag) |
| } |
| |
| func releaseViewURL(tag string) string { |
| return fmt.Sprintf("https://github.com/%s/%s/releases/tag/%s", *githubOrg, *githubRepo, tag) |
| } |
| |
| func releaseEditURL(tag string) string { |
| return fmt.Sprintf("https://github.com/%s/%s/releases/edit/%s", *githubOrg, *githubRepo, tag) |
| } |
| |
| func fetch(url string) (*http.Response, error) { |
| resp, err := http.Get(url) |
| if err != nil { |
| return nil, err |
| } |
| if resp.StatusCode != 200 { |
| resp.Body.Close() |
| return nil, fmt.Errorf("got status code of %d from %q instead of 200", resp.StatusCode, url) |
| } |
| return resp, nil |
| } |
| |
| type releaseFetchError struct{ error } |
| type releaseMismatchError struct{ error } |
| |
| func sha256Reader(r io.Reader) ([]byte, error) { |
| h := sha256.New() |
| if _, err := io.Copy(h, r); err != nil { |
| return nil, err |
| } |
| return h.Sum(nil), nil |
| } |
| |
| func run(tag string) error { |
| // Check the tag does not contain any characters that would break the URL |
| // or filesystem. |
| for _, c := range tag { |
| if c != '.' && !('0' <= c && c <= '9') && !('a' <= c && c <= 'z') && !('A' <= c && c <= 'Z') { |
| return fmt.Errorf("invalid tag %q", tag) |
| } |
| } |
| |
| // Read the tag from git. We will use this to ensure the archive is correct. |
| var expectedTree []treeEntry |
| if err := step("Hashing tree from git", func(s *stepPrinter) error { |
| var err error |
| expectedTree, err = gitHashTree(s, tag) |
| return err |
| }); err != nil { |
| return err |
| } |
| |
| // Hash the archive tarball. |
| // |
| // BCR does not accept archive tarballs, due to concerns that GitHub may |
| // change the hash, and instead prefers release tarballs. Release tarballs, |
| // however, are uploaded by individual developers, with no guaranteed they |
| // match the contents of the tag. |
| // |
| // This script checks the release tarball against the tag in the on-disk git |
| // repository, so we validate the contents independent of GitHub. We |
| // additionally check that release tarball matches the archive tarball. The |
| // archive tarballs are stable in practice, and this is an easy, though |
| // still GitHub-dependent, property that anyone can check. (This script |
| // assumes GitHub did not change their tarballs in the short window between |
| // when the release tarball was uploaded and this script runs.) |
| var archiveSHA256 []byte |
| if !*skipArchiveCheck { |
| if err := step("Fetching archive tarball", func(s *stepPrinter) error { |
| archive, err := fetch(archiveURL(tag)) |
| if err != nil { |
| return err |
| } |
| defer archive.Body.Close() |
| archiveSHA256, err = sha256Reader(s.httpBodyWithProgress(archive)) |
| return err |
| }); err != nil { |
| return err |
| } |
| } |
| |
| // Prepare an output directory. |
| var dir string |
| var err error |
| if len(*outDir) != 0 { |
| dir, err = filepath.Abs(*outDir) |
| } else { |
| dir, err = os.MkdirTemp("", "boringssl_bcr") |
| } |
| if err != nil { |
| return err |
| } |
| |
| // Fetch the release tarball. As we stream it, we do three things: |
| // |
| // 1. Compute the overall SHA-256 sum. This hash must be saved in the BCR |
| // configuration. |
| // |
| // 2. Hash the contents of each file in the tarball, to compare against the |
| // contents in git. |
| // |
| // 3. Extract MODULE.bazel and presubmit.yml, to save in the temporary |
| // directory. This is needed to work around limitations in BCR's tooling. |
| // See https://github.com/bazelbuild/bazel-central-registry/issues/2781 |
| var releaseTree []treeEntry |
| releaseHash := sha256.New() |
| stripPrefix := fmt.Sprintf("%s-%s/", *githubRepo, tag) |
| if err := step("Fetching release tarball", func(s *stepPrinter) error { |
| release, err := fetch(releaseURL(tag)) |
| if err != nil { |
| return releaseFetchError{err} |
| } |
| defer release.Body.Close() |
| |
| // Hash the tarball as we read it. |
| reader := s.httpBodyWithProgress(release) |
| reader = io.TeeReader(reader, releaseHash) |
| |
| zlibReader, err := gzip.NewReader(reader) |
| if err != nil { |
| return fmt.Errorf("error reading release tarball: %w", err) |
| } |
| |
| tarReader := tar.NewReader(zlibReader) |
| var seenModule, seenPresubmit bool |
| for { |
| header, err := tarReader.Next() |
| if err == io.EOF { |
| break |
| } |
| if err != nil { |
| return fmt.Errorf("error reading release tarball: %w", err) |
| } |
| |
| var mode treeEntryMode |
| var fileReader io.Reader |
| switch header.Typeflag { |
| case tar.TypeDir: |
| // Check directories have a suitable prefix, but otherwise ignore |
| // them. |
| if !strings.HasPrefix(header.Name, stripPrefix) { |
| return fmt.Errorf("release tarball contained path %q which did not begin with %q", header.Name, stripPrefix) |
| } |
| continue |
| case tar.TypeXGlobalHeader: |
| continue |
| case tar.TypeReg: |
| if header.Mode&1 != 0 { |
| mode = treeEntryExecutable |
| } else { |
| mode = treeEntryRegular |
| } |
| fileReader = tarReader |
| case tar.TypeSymlink: |
| mode = treeEntrySymlink |
| fileReader = strings.NewReader(header.Linkname) |
| default: |
| return fmt.Errorf("path %q in release archive had unknown type %d", header.Name, header.Typeflag) |
| } |
| |
| path, ok := strings.CutPrefix(header.Name, stripPrefix) |
| if !ok { |
| return fmt.Errorf("release tarball contained path %q which did not begin with %q", header.Name, stripPrefix) |
| } |
| |
| var saveFile *os.File |
| if mode == treeEntryRegular && path == "MODULE.bazel" { |
| if seenModule { |
| return fmt.Errorf("release tarball contained duplicate MODULE.bazel file") |
| } |
| saveFile, err = os.Create(filepath.Join(dir, "MODULE.bazel")) |
| if err != nil { |
| return err |
| } |
| seenModule = true |
| } else if mode == treeEntryRegular && path == ".bcr/presubmit.yml" { |
| if seenPresubmit { |
| return fmt.Errorf("release tarball contained duplicate .bcr/presubmit.yml file") |
| } |
| saveFile, err = os.Create(filepath.Join(dir, "presubmit.yml")) |
| if err != nil { |
| return err |
| } |
| seenPresubmit = true |
| } |
| |
| if saveFile != nil { |
| fileReader = io.TeeReader(fileReader, saveFile) |
| } |
| |
| sha256, err := sha256Reader(fileReader) |
| saveFile.Close() |
| if err != nil { |
| return fmt.Errorf("error reading %q in release archive: %w", header.Name, err) |
| } |
| |
| releaseTree = append(releaseTree, treeEntry{path: path, mode: mode, sha256: sha256}) |
| } |
| |
| sortTree(releaseTree) |
| |
| // Check the zlib checksum is correct. |
| if err := zlibReader.Close(); err != nil { |
| return fmt.Errorf("error reading release tarball: %w", err) |
| } |
| |
| // Ensure we have read (and thus hashed) the entire archive. |
| if _, err := io.Copy(io.Discard, reader); err != nil { |
| return fmt.Errorf("error reading release archive: %w", err) |
| } |
| |
| if !seenModule && len(*moduleOverride) == 0 { |
| return fmt.Errorf("could not find MODULE.bazel in release tarball") |
| } |
| if !seenPresubmit && len(*presubmitOverride) == 0 { |
| return fmt.Errorf("could not find .bcr/presubmit.yml in release tarball") |
| } |
| return nil |
| }); err != nil { |
| return err |
| } |
| |
| releaseSHA256 := releaseHash.Sum(nil) |
| if !*skipArchiveCheck && !bytes.Equal(archiveSHA256, releaseSHA256) { |
| return releaseMismatchError{fmt.Errorf("release hash was %x, which did not match archive hash was %x", archiveSHA256, releaseSHA256)} |
| } |
| |
| if err := compareTrees(releaseTree, expectedTree); err != nil { |
| return err |
| } |
| |
| config := bcrConfig{ |
| Name: *moduleName, |
| Version: tag, |
| CompatibilityLevel: *compatibilityLevel, |
| ModuleDotBazel: ptr(filepath.Join(dir, "MODULE.bazel")), |
| URL: ptr(releaseURL(tag)), |
| StripPrefix: &stripPrefix, |
| PresubmitYml: ptr(filepath.Join(dir, "presubmit.yml")), |
| // encoding/json will encode nil slices as null instead of the empty array. |
| Deps: []string{}, |
| Patches: []string{}, |
| BuildTargets: []string{}, |
| TestModuleBuildTargets: []string{}, |
| TestModuleTestTargets: []string{}, |
| } |
| |
| if len(*moduleOverride) != 0 { |
| override, err := filepath.Abs(*moduleOverride) |
| if err != nil { |
| return err |
| } |
| config.ModuleDotBazel = &override |
| } |
| if len(*presubmitOverride) != 0 { |
| override, err := filepath.Abs(*presubmitOverride) |
| if err != nil { |
| return err |
| } |
| config.PresubmitYml = &override |
| } |
| |
| configJSON, err := json.Marshal(config) |
| if err != nil { |
| return err |
| } |
| |
| jsonPath := filepath.Join(dir, "bcr.json") |
| if err := os.WriteFile(jsonPath, configJSON, 0666); err != nil { |
| return err |
| } |
| |
| fmt.Printf("\n") |
| fmt.Printf("BCR configuration written to %q\n", dir) |
| fmt.Printf("\n") |
| fmt.Printf("Clone the BCR repository at:\n") |
| fmt.Printf(" https://github.com/bazelbuild/bazel-central-registry\n") |
| fmt.Printf("\n") |
| fmt.Printf("Then, run the following command to prepare the module update:\n") |
| fmt.Printf(" bazelisk run //tools:add_module -- --input %s\n", jsonPath) |
| fmt.Printf("\n") |
| fmt.Printf("Finally, commit the result and send the BCR repository a PR.\n") |
| return nil |
| } |
| |
| func main() { |
| flag.Usage = func() { |
| fmt.Fprint(os.Stderr, "Usage: go run ./util/prepare_bcr_module [FLAGS...] TAG\n") |
| flag.PrintDefaults() |
| } |
| flag.Parse() |
| if flag.NArg() != 1 { |
| fmt.Fprintf(os.Stderr, "Expected exactly one tag specified.\n") |
| flag.Usage() |
| os.Exit(1) |
| } |
| |
| tag := flag.Arg(0) |
| if err := run(tag); err != nil { |
| if _, ok := err.(releaseFetchError); ok { |
| fmt.Fprintf(os.Stderr, "Error fetching release URL for %q: %s\n", tag, err) |
| fmt.Fprintf(os.Stderr, "\n") |
| fmt.Fprintf(os.Stderr, "To fix this, follow the following steps:\n") |
| fmt.Fprintf(os.Stderr, "1. Open %s in a browser.\n", releaseViewURL(tag)) |
| fmt.Fprintf(os.Stderr, "2. Download the \"Source code (tar.gz)\" archive.\n") |
| fmt.Fprintf(os.Stderr, "3. Click the edit icon, or open %s in your browser.\n", releaseEditURL(tag)) |
| fmt.Fprintf(os.Stderr, "4. Attach the downloaded boringssl-%s.tar.gz to the release.\n", tag) |
| fmt.Fprintf(os.Stderr, "\n") |
| } else if _, ok := err.(releaseMismatchError); ok { |
| fmt.Fprintf(os.Stderr, "Invalid release tarball for %q: %s\n", tag, err) |
| fmt.Fprintf(os.Stderr, "\n") |
| fmt.Fprintf(os.Stderr, "To fix this, follow the following steps:\n") |
| fmt.Fprintf(os.Stderr, "1. Open %s in a browser.\n", releaseViewURL(tag)) |
| fmt.Fprintf(os.Stderr, "2. Download the \"Source code (tar.gz)\" archive.\n") |
| fmt.Fprintf(os.Stderr, "3. Click the edit icon, or open %s in your browser.\n", releaseEditURL(tag)) |
| fmt.Fprintf(os.Stderr, "4. Delete the old boringssl-%s.tar.gz from the release.\n", tag) |
| fmt.Fprintf(os.Stderr, "5. Re-attach the downloaded boringssl-%s.tar.gz to the release.\n", tag) |
| fmt.Fprintf(os.Stderr, "\n") |
| } else { |
| fmt.Fprintf(os.Stderr, "Error preparing release %q: %s\n", tag, err) |
| } |
| os.Exit(1) |
| } |
| } |
